diff options
Diffstat (limited to '')
27 files changed, 1490 insertions, 0 deletions
diff --git a/toolkit/components/telemetry/tests/marionette/harness/MANIFEST.in b/toolkit/components/telemetry/tests/marionette/harness/MANIFEST.in new file mode 100644 index 0000000000..e24a6b1ba6 --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/harness/MANIFEST.in @@ -0,0 +1,3 @@ +exclude MANIFEST.in +include requirements.txt +recursive-include telemetry_harness/resources *
\ No newline at end of file diff --git a/toolkit/components/telemetry/tests/marionette/harness/requirements.txt b/toolkit/components/telemetry/tests/marionette/harness/requirements.txt new file mode 100644 index 0000000000..a95e794fe0 --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/harness/requirements.txt @@ -0,0 +1,2 @@ +marionette-harness >= 4.0.0 +requests==2.11.1
\ No newline at end of file diff --git a/toolkit/components/telemetry/tests/marionette/harness/setup.py b/toolkit/components/telemetry/tests/marionette/harness/setup.py new file mode 100644 index 0000000000..f650eb18ed --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/harness/setup.py @@ -0,0 +1,48 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +from setuptools import setup, find_packages + +PACKAGE_VERSION = "0.1" + +THIS_DIR = os.path.dirname(os.path.realpath(__name__)) + + +def read(*parts): + with open(os.path.join(THIS_DIR, *parts)) as f: + return f.read() + + +setup( + name="telemetry-harness", + version=PACKAGE_VERSION, + description=( + "Custom Marionette runner classes and entry scripts for " + "Telemetry specific Marionette tests." + ), + classifiers=[ + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Topic :: Software Development :: Libraries :: Python Modules", + ], + keywords="mozilla", + author="Firefox Test Engineering Team", + author_email="firefox-test-engineering@mozilla.org", + url="https://developer.mozilla.org/en-US/docs/Mozilla/QA/telemetry_harness", + license="MPL 2.0", + packages=find_packages(), + include_package_data=True, + zip_safe=False, + install_requires=read("requirements.txt").splitlines(), + entry_points=""" + [console_scripts] + telemetry-harness = telemetry_harness.runtests:cli + """, +) diff --git a/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/__init__.py b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/__init__.py new file mode 100644 index 0000000000..6fbe8159b2 --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/fog_ping_filters.py b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/fog_ping_filters.py new file mode 100644 index 0000000000..9db0f5a72f --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/fog_ping_filters.py @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +class FOGPingFilter(object): + """Ping filter that accepts any FOG pings.""" + + def __call__(self, ping): + return True + + +class FOGDocTypePingFilter(FOGPingFilter): + """Ping filter that accepts FOG pings that match the doc-type.""" + + def __init__(self, doc_type): + super(FOGDocTypePingFilter, self).__init__() + self.doc_type = doc_type + + def __call__(self, ping): + if not super(FOGDocTypePingFilter, self).__call__(ping): + return False + + # Verify that the given ping was submitted to the URL for the doc_type + return ping["request_url"]["doc_type"] == self.doc_type + + +FOG_DELETION_REQUEST_PING = FOGDocTypePingFilter("deletion-request") +FOG_ONE_PING_ONLY_PING = FOGDocTypePingFilter("one-ping-only") diff --git a/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/fog_ping_server.py b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/fog_ping_server.py new file mode 100644 index 0000000000..7d84b40d77 --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/fog_ping_server.py @@ -0,0 +1,77 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import zlib + +from marionette_harness.runner import httpd +from mozlog import get_default_logger +from six.moves.urllib import parse as urlparse + + +class FOGPingServer(object): + """HTTP server for receiving Firefox on Glean pings.""" + + def __init__(self, server_root, url): + self._logger = get_default_logger(component="fog_ping_server") + self.pings = [] + + @httpd.handlers.handler + def pings_handler(request, response): + """Handler for HTTP requests to the ping server.""" + request_data = request.body + + if request.headers.get("Content-Encoding") == "gzip": + request_data = zlib.decompress(request_data, zlib.MAX_WBITS | 16) + + request_url = request.route_match.copy() + + self.pings.append( + { + "request_url": request_url, + "payload": json.loads(request_data), + "debug_tag": request.headers.get("X-Debug-ID"), + } + ) + + self._logger.info( + "pings_handler received '{}' ping".format(request_url["doc_type"]) + ) + + status_code = 200 + content = "OK" + headers = [ + ("Content-Type", "text/plain"), + ("Content-Length", len(content)), + ] + + return (status_code, headers, content) + + self._httpd = httpd.FixtureServer(server_root, url=url) + + # See https://mozilla.github.io/glean/book/user/pings/index.html#ping-submission + self._httpd.router.register( + "POST", + "/submit/{application_id}/{doc_type}/{glean_schema_version}/{document_id}", + pings_handler, + ) + + @property + def url(self): + """Return the URL for the running HTTP FixtureServer.""" + return self._httpd.get_url("/") + + @property + def port(self): + """Return the port for the running HTTP FixtureServer.""" + parse_result = urlparse.urlparse(self.url) + return parse_result.port + + def start(self): + """Start the HTTP FixtureServer.""" + return self._httpd.start() + + def stop(self): + """Stop the HTTP FixtureServer.""" + return self._httpd.stop() diff --git a/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/fog_testcase.py b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/fog_testcase.py new file mode 100644 index 0000000000..fad2781a80 --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/fog_testcase.py @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozlog +from telemetry_harness.fog_ping_server import FOGPingServer +from telemetry_harness.testcase import TelemetryTestCase + + +class FOGTestCase(TelemetryTestCase): + """Base testcase class for project FOG.""" + + def __init__(self, *args, **kwargs): + """Initialize the test case and create a ping server.""" + super(FOGTestCase, self).__init__(*args, **kwargs) + self._logger = mozlog.get_default_logger(component="FOGTestCase") + + def setUp(self, *args, **kwargs): + """Set up the test case and create a FOG ping server. + + This test is skipped if the build doesn't support FOG. + """ + super(FOGTestCase, self).setUp(*args, **kwargs) + + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + fog_present = self.marionette.execute_script( + "return AppConstants.MOZ_GLEAN;" + ) + + if not fog_present: + # Before we skip this test, we need to quit marionette and the ping + # server created in TelemetryTestCase by running tearDown + super(FOGTestCase, self).tearDown(*args, **kwargs) + self.skipTest("FOG is only present in AppConstants.MOZ_GLEAN builds.") + + self.fog_ping_server = FOGPingServer( + self.testvars["server_root"], "http://localhost:0" + ) + self.fog_ping_server.start() + + self._logger.info( + "Submitting to FOG ping server at {}".format(self.fog_ping_server.url) + ) + + self.marionette.enforce_gecko_prefs( + { + "telemetry.fog.test.localhost_port": self.fog_ping_server.port, + } + ) + + def tearDown(self, *args, **kwargs): + super(FOGTestCase, self).tearDown(*args, **kwargs) + self.fog_ping_server.stop() diff --git a/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/ping_filters.py b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/ping_filters.py new file mode 100644 index 0000000000..6e003b25d5 --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/ping_filters.py @@ -0,0 +1,75 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +class PingFilter(object): + """Ping filter that accepts any pings.""" + + def __call__(self, ping): + return True + + +class DeletionRequestPingFilter(PingFilter): + """Ping filter that accepts deletion-request pings.""" + + def __call__(self, ping): + if not super(DeletionRequestPingFilter, self).__call__(ping): + return False + + return ping["type"] == "deletion-request" + + +class EventPingFilter(PingFilter): + """Ping filter that accepts event pings.""" + + def __call__(self, ping): + if not super(EventPingFilter, self).__call__(ping): + return False + + return ping["type"] == "event" + + +class FirstShutdownPingFilter(PingFilter): + """Ping filter that accepts first-shutdown pings.""" + + def __call__(self, ping): + if not super(FirstShutdownPingFilter, self).__call__(ping): + return False + + return ping["type"] == "first-shutdown" + + +class MainPingFilter(PingFilter): + """Ping filter that accepts main pings.""" + + def __call__(self, ping): + if not super(MainPingFilter, self).__call__(ping): + return False + + return ping["type"] == "main" + + +class MainPingReasonFilter(MainPingFilter): + """Ping filter that accepts main pings that match the + specified reason. + """ + + def __init__(self, reason): + super(MainPingReasonFilter, self).__init__() + self.reason = reason + + def __call__(self, ping): + if not super(MainPingReasonFilter, self).__call__(ping): + return False + + return ping["payload"]["info"]["reason"] == self.reason + + +ANY_PING = PingFilter() +DELETION_REQUEST_PING = DeletionRequestPingFilter() +EVENT_PING = EventPingFilter() +FIRST_SHUTDOWN_PING = FirstShutdownPingFilter() +MAIN_PING = MainPingFilter() +MAIN_SHUTDOWN_PING = MainPingReasonFilter("shutdown") +MAIN_ENVIRONMENT_CHANGE_PING = MainPingReasonFilter("environment-change") diff --git a/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/ping_server.py b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/ping_server.py new file mode 100644 index 0000000000..d05c265504 --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/ping_server.py @@ -0,0 +1,65 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import zlib + +import mozlog + +from marionette_harness.runner import httpd + + +class PingServer(object): + """HTTP server for receiving Firefox Client Telemetry pings.""" + + def __init__(self, server_root, url): + self._logger = mozlog.get_default_logger(component="pingserver") + self.pings = [] + + @httpd.handlers.handler + def pings_handler(request, response): + """Handler for HTTP requests to the ping server.""" + request_data = request.body + + if request.headers.get("Content-Encoding") == "gzip": + request_data = zlib.decompress(request_data, zlib.MAX_WBITS | 16) + + ping_data = json.loads(request_data) + + # Store JSON data to self.pings to be used by wait_for_pings() + self.pings.append(ping_data) + + ping_type = ping_data["type"] + + log_message = "pings_handler received '{}' ping".format(ping_type) + + if ping_type == "main": + ping_reason = ping_data["payload"]["info"]["reason"] + log_message = "{} with reason '{}'".format(log_message, ping_reason) + + self._logger.info(log_message) + + status_code = 200 + content = "OK" + headers = [ + ("Content-Type", "text/plain"), + ("Content-Length", len(content)), + ] + + return (status_code, headers, content) + + self._httpd = httpd.FixtureServer(server_root, url=url) + self._httpd.router.register("POST", "/pings*", pings_handler) + + def get_url(self, *args, **kwargs): + """Return a URL from the HTTP server.""" + return self._httpd.get_url(*args, **kwargs) + + def start(self): + """Start the HTTP server.""" + return self._httpd.start() + + def stop(self): + """Stop the HTTP server.""" + return self._httpd.stop() diff --git a/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/resources/helloworld/helloworld.html b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/resources/helloworld/helloworld.html new file mode 100644 index 0000000000..146ad025d9 --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/resources/helloworld/helloworld.html @@ -0,0 +1,18 @@ +<!DOCTYPE html> +<html> + <head> + <meta charset="utf-8"> + <style> + body { + background-color: lightgrey; + } + p { + font-size: 25px; + padding: 25px 50px; + } + </style> + </head> + <body> + <p>Hello World!</p> + </body> +</html> diff --git a/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/resources/helloworld/manifest.json b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/resources/helloworld/manifest.json new file mode 100644 index 0000000000..0e35d8a2e3 --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/resources/helloworld/manifest.json @@ -0,0 +1,12 @@ +{ + "description": "Extension to be installed in telemetry-tests-client tests.", + "manifest_version": 2, + "name": "helloworld", + "version": "1.0", + "homepage_url": "https://hg.mozilla.org/mozilla-central/", + "browser_action": { + "browser_style": true, + "default_title": "Hello World", + "default_popup": "helloworld.html" + } +} diff --git a/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/runner.py b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/runner.py new file mode 100644 index 0000000000..d95b17c03c --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/runner.py @@ -0,0 +1,56 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from marionette_harness import BaseMarionetteTestRunner +from testcase import TelemetryTestCase + +SERVER_URL = "http://localhost:8000" + + +class TelemetryTestRunner(BaseMarionetteTestRunner): + """TestRunner for the telemetry-tests-client suite.""" + + def __init__(self, **kwargs): + """Set test variables and preferences specific to Firefox client + telemetry. + """ + + # Select the appropriate GeckoInstance + kwargs["app"] = "fxdesktop" + + prefs = kwargs.pop("prefs", {}) + + # Set Firefox Client Telemetry specific preferences + prefs.update( + { + # Clear the region detection url to + # * avoid net access in tests + # * stabilize browser.search.region to avoid extra subsessions (bug 1579840#c40) + "browser.region.network.url": "", + # Disable smart sizing because it changes prefs at startup. (bug 1547750) + "browser.cache.disk.smart_size.enabled": False, + "toolkit.telemetry.server": "{}/pings".format(SERVER_URL), + "telemetry.fog.test.localhost_port": -1, + "toolkit.telemetry.initDelay": 1, + "toolkit.telemetry.minSubsessionLength": 0, + "datareporting.healthreport.uploadEnabled": True, + "datareporting.policy.dataSubmissionEnabled": True, + "datareporting.policy.dataSubmissionPolicyBypassNotification": True, + "toolkit.telemetry.log.level": "Trace", + "toolkit.telemetry.log.dump": True, + "toolkit.telemetry.send.overrideOfficialCheck": True, + "toolkit.telemetry.testing.disableFuzzingDelay": True, + # Disable Normandy to avoid extra subsessions due to Experiment + # activation in tests (bug 1641571) + "app.normandy.enabled": False, + } + ) + + super(TelemetryTestRunner, self).__init__(prefs=prefs, **kwargs) + + self.testvars["server_root"] = kwargs["server_root"] + self.testvars["server_url"] = SERVER_URL + + self.test_handlers = [TelemetryTestCase] diff --git a/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/runtests.py b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/runtests.py new file mode 100644 index 0000000000..9b4b8872c9 --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/runtests.py @@ -0,0 +1,14 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from marionette_harness.runtests import cli as mn_cli +from runner import TelemetryTestRunner + + +def cli(args=None): + mn_cli(runner_class=TelemetryTestRunner, args=args) + + +if __name__ == "__main__": + cli() diff --git a/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/testcase.py b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/testcase.py new file mode 100644 index 0000000000..b4646e6565 --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/testcase.py @@ -0,0 +1,242 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import contextlib +import os +import re +import textwrap + +from marionette_driver.addons import Addons +from marionette_driver.errors import MarionetteException +from marionette_driver.wait import Wait +from marionette_driver import By, keys +from marionette_harness import MarionetteTestCase +from marionette_harness.runner.mixins.window_manager import WindowManagerMixin + +from ping_server import PingServer + + +CANARY_CLIENT_ID = "c0ffeec0-ffee-c0ff-eec0-ffeec0ffeec0" +UUID_PATTERN = re.compile( + r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" +) + + +class TelemetryTestCase(WindowManagerMixin, MarionetteTestCase): + def __init__(self, *args, **kwargs): + """Initialize the test case and create a ping server.""" + super(TelemetryTestCase, self).__init__(*args, **kwargs) + + self.ping_server = PingServer( + self.testvars["server_root"], self.testvars["server_url"] + ) + + def setUp(self, *args, **kwargs): + """Set up the test case and start the ping server.""" + super(TelemetryTestCase, self).setUp(*args, **kwargs) + + # Store IDs of addons installed via self.install_addon() + self.addon_ids = [] + + with self.marionette.using_context(self.marionette.CONTEXT_CONTENT): + self.marionette.navigate("about:about") + + self.ping_server.start() + + def disable_telemetry(self): + """Disable the Firefox Data Collection and Use in the current browser.""" + self.marionette.instance.profile.set_persistent_preferences( + {"datareporting.healthreport.uploadEnabled": False} + ) + self.marionette.set_pref("datareporting.healthreport.uploadEnabled", False) + + def enable_telemetry(self): + """Enable the Firefox Data Collection and Use in the current browser.""" + self.marionette.instance.profile.set_persistent_preferences( + {"datareporting.healthreport.uploadEnabled": True} + ) + self.marionette.set_pref("datareporting.healthreport.uploadEnabled", True) + + @contextlib.contextmanager + def new_tab(self): + """Perform operations in a new tab and then close the new tab.""" + + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + start_tab = self.marionette.current_window_handle + new_tab = self.open_tab(focus=True) + self.marionette.switch_to_window(new_tab) + + yield + + self.marionette.close() + self.marionette.switch_to_window(start_tab) + + def search(self, text): + """Perform a search via the browser's URL bar.""" + + # Reload newtab to prevent urlbar from not accepting correct input + with self.marionette.using_context(self.marionette.CONTEXT_CONTENT): + self.marionette.navigate("about:newtab") + + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + self.marionette.execute_script("gURLBar.select();") + urlbar = self.marionette.find_element(By.ID, "urlbar-input") + urlbar.send_keys(keys.Keys.DELETE) + urlbar.send_keys(text + keys.Keys.ENTER) + # This script checks that the search terms used for searching + # appear in the URL when the page loads. + script = """\ + let location = document.location.toString() + function validate(term){ + return location.includes(term) + } + return arguments[0].every(validate) + """ + # Wait for search page to load + with self.marionette.using_context(self.marionette.CONTEXT_CONTENT): + Wait(self.marionette, 30, 0.5).until( + lambda driver: driver.execute_script( + script, script_args=[text.split()] + ), + message="Search terms not found, maybe the page didn't load?", + ) + + def search_in_new_tab(self, text): + """Open a new tab and perform a search via the browser's URL bar, + then close the new tab.""" + + with self.new_tab(): + self.search(text) + + def assertIsValidUUID(self, value): + """Check if the given UUID is valid.""" + + self.assertIsNotNone(value) + self.assertNotEqual(value, "") + + # Check for client ID that is used when Telemetry upload is disabled + self.assertNotEqual(value, CANARY_CLIENT_ID, msg="UUID is CANARY CLIENT ID") + + self.assertIsNotNone( + re.match(UUID_PATTERN, value), + msg="UUID does not match regular expression", + ) + + def wait_for_pings(self, action_func, ping_filter, count, ping_server=None): + """Call the given action and wait for pings to come in and return + the `count` number of pings, that match the given filter. + """ + + if ping_server is None: + ping_server = self.ping_server + + # Keep track of the current number of pings + current_num_pings = len(ping_server.pings) + + # New list to store new pings that satisfy the filter + filtered_pings = [] + + def wait_func(*args, **kwargs): + # Ignore existing pings in ping_server.pings + new_pings = ping_server.pings[current_num_pings:] + + # Filter pings to make sure we wait for the correct ping type + filtered_pings[:] = [p for p in new_pings if ping_filter(p)] + + return len(filtered_pings) >= count + + self.logger.info( + "wait_for_pings running action '{action}'.".format( + action=action_func.__name__ + ) + ) + + # Call given action and wait for a ping + action_func() + + try: + Wait(self.marionette, 60).until(wait_func) + except Exception as e: + self.fail("Error waiting for ping: {}".format(e.message)) + + return filtered_pings[:count] + + def wait_for_ping(self, action_func, ping_filter, ping_server=None): + """Call wait_for_pings() with the given action_func and ping_filter and + return the first result. + """ + [ping] = self.wait_for_pings( + action_func, ping_filter, 1, ping_server=ping_server + ) + return ping + + def restart_browser(self): + """Restarts browser while maintaining the same profile.""" + return self.marionette.restart(clean=False, in_app=True) + + def start_browser(self): + """Start the browser.""" + return self.marionette.start_session() + + def quit_browser(self): + """Quit the browser.""" + return self.marionette.quit(in_app=True) + + def install_addon(self): + """Install a minimal addon and add its ID to self.addon_ids.""" + + resources_dir = os.path.join(os.path.dirname(__file__), "resources") + addon_path = os.path.abspath(os.path.join(resources_dir, "helloworld")) + + try: + # Ensure the Environment has init'd so the installed addon + # triggers an "environment-change" ping. + script = """\ + let [resolve] = arguments; + Cu.import("resource://gre/modules/TelemetryEnvironment.jsm"); + TelemetryEnvironment.onInitialized().then(resolve); + """ + + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + self.marionette.execute_async_script(textwrap.dedent(script)) + + addons = Addons(self.marionette) + addon_id = addons.install(addon_path, temp=True) + except MarionetteException as e: + self.fail("{} - Error installing addon: {} - ".format(e.cause, e.message)) + else: + self.addon_ids.append(addon_id) + + def set_persistent_profile_preferences(self, preferences): + """Wrapper for setting persistent preferences on a user profile""" + return self.marionette.instance.profile.set_persistent_preferences(preferences) + + def set_preferences(self, preferences): + """Wrapper for setting persistent preferences on a user profile""" + return self.marionette.set_prefs(preferences) + + @property + def client_id(self): + """Return the ID of the current client.""" + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + return self.marionette.execute_script( + 'Cu.import("resource://gre/modules/ClientID.jsm");' + "return ClientID.getCachedClientID();" + ) + + @property + def subsession_id(self): + """Return the ID of the current subsession.""" + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + ping_data = self.marionette.execute_script( + 'Cu.import("resource://gre/modules/TelemetryController.jsm");' + "return TelemetryController.getCurrentPingData(true);" + ) + return ping_data[u"payload"][u"info"][u"subsessionId"] + + def tearDown(self, *args, **kwargs): + """Stop the ping server and tear down the testcase.""" + super(TelemetryTestCase, self).tearDown() + self.ping_server.stop() + self.marionette.quit(clean=True) diff --git a/toolkit/components/telemetry/tests/marionette/mach_commands.py b/toolkit/components/telemetry/tests/marionette/mach_commands.py new file mode 100644 index 0000000000..783563baa3 --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/mach_commands.py @@ -0,0 +1,101 @@ +# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+import logging
+import os
+import sys
+
+from mach.decorators import CommandProvider, Command
+
+from mozbuild.base import (
+ MachCommandBase,
+ MachCommandConditions as conditions,
+ BinaryNotFoundException,
+)
+
+
+def create_parser_tests():
+ from marionette_harness.runtests import MarionetteArguments
+ from mozlog.structured import commandline
+
+ parser = MarionetteArguments()
+ commandline.add_logging_group(parser)
+ return parser
+
+
+def run_telemetry(tests, binary=None, topsrcdir=None, **kwargs):
+ from mozlog.structured import commandline
+
+ from telemetry_harness.runtests import TelemetryTestRunner
+
+ from marionette_harness.runtests import MarionetteHarness
+
+ parser = create_parser_tests()
+
+ if not tests:
+ tests = [
+ os.path.join(
+ topsrcdir,
+ "toolkit/components/telemetry/tests/marionette/tests/manifest.ini",
+ )
+ ]
+
+ args = argparse.Namespace(tests=tests)
+
+ args.binary = binary
+ args.logger = kwargs.pop("log", None)
+
+ for k, v in kwargs.iteritems():
+ setattr(args, k, v)
+
+ parser.verify_usage(args)
+
+ os.environ["MOZ_IGNORE_NSS_SHUTDOWN_LEAKS"] = "1"
+
+ if not args.logger:
+ args.logger = commandline.setup_logging(
+ "Telemetry Client Tests", args, {"mach": sys.stdout}
+ )
+ failed = MarionetteHarness(TelemetryTestRunner, args=vars(args)).run()
+ if failed > 0:
+ return 1
+ return 0
+
+
+@CommandProvider
+class TelemetryTest(MachCommandBase):
+ @Command(
+ "telemetry-tests-client",
+ category="testing",
+ description="Run tests specifically for the Telemetry client",
+ conditions=[conditions.is_firefox_or_android],
+ parser=create_parser_tests,
+ )
+ def telemetry_test(self, tests, **kwargs):
+ if "test_objects" in kwargs:
+ tests = []
+ for obj in kwargs["test_objects"]:
+ tests.append(obj["file_relpath"])
+ del kwargs["test_objects"]
+ if not kwargs.get("binary") and conditions.is_firefox(self):
+ try:
+ kwargs["binary"] = self.get_binary_path("app")
+ except BinaryNotFoundException as e:
+ self.log(
+ logging.ERROR,
+ "telemetry-tests-client",
+ {"error": str(e)},
+ "ERROR: {error}",
+ )
+ self.log(
+ logging.INFO, "telemetry-tests-client", {"help": e.help()}, "{help}"
+ )
+ return 1
+ if not kwargs.get("server_root"):
+ kwargs[
+ "server_root"
+ ] = "toolkit/components/telemetry/tests/marionette/harness/www"
+ return run_telemetry(tests, topsrcdir=self.topsrcdir, **kwargs)
diff --git a/toolkit/components/telemetry/tests/marionette/moz.build b/toolkit/components/telemetry/tests/marionette/moz.build new file mode 100644 index 0000000000..7a6aa0bc15 --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/moz.build @@ -0,0 +1,11 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +TELEMETRY_TESTS_CLIENT_MANIFESTS += ["tests/manifest.ini"] + +with Files("**"): + BUG_COMPONENT = ("Toolkit", "Telemetry") + SCHEDULES.exclusive = ["telemetry-tests-client"] diff --git a/toolkit/components/telemetry/tests/marionette/tests/client/manifest.ini b/toolkit/components/telemetry/tests/marionette/tests/client/manifest.ini new file mode 100644 index 0000000000..ec5d061640 --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/tests/client/manifest.ini @@ -0,0 +1,10 @@ +[DEFAULT] +tags = client + +[test_deletion_request_ping.py] +[test_event_ping.py] +[test_main_tab_scalars.py] +[test_search_counts_across_sessions.py] +[test_subsession_management.py] +[test_fog_deletion_request_ping.py] +[test_fog_custom_ping.py] diff --git a/toolkit/components/telemetry/tests/marionette/tests/client/test_deletion_request_ping.py b/toolkit/components/telemetry/tests/marionette/tests/client/test_deletion_request_ping.py new file mode 100644 index 0000000000..cb07c6b2db --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/tests/client/test_deletion_request_ping.py @@ -0,0 +1,64 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from telemetry_harness.testcase import TelemetryTestCase +from telemetry_harness.ping_filters import ( + ANY_PING, + DELETION_REQUEST_PING, + MAIN_SHUTDOWN_PING, +) + + +class TestDeletionRequestPing(TelemetryTestCase): + """Tests for "deletion-request" ping.""" + + def test_deletion_request_ping_across_sessions(self): + """Test the "deletion-request" ping behaviour across sessions.""" + + # Get the client_id. + client_id = self.wait_for_ping(self.install_addon, ANY_PING)["clientId"] + self.assertIsValidUUID(client_id) + + # Trigger an "deletion-request" ping. + ping = self.wait_for_ping(self.disable_telemetry, DELETION_REQUEST_PING) + + self.assertIn("clientId", ping) + self.assertIn("payload", ping) + self.assertNotIn("environment", ping["payload"]) + + # Close Firefox cleanly. + self.quit_browser() + + # TODO: Check pending pings aren't persisted + + # Start Firefox. + self.start_browser() + + # Trigger an environment change, which isn't allowed to send a ping. + self.install_addon() + + # Ensure we've sent no pings since "disabling telemetry". + self.assertEqual(self.ping_server.pings[-1], ping) + + # Turn Telemetry back on. + self.enable_telemetry() + + # Close Firefox cleanly, collecting its "main"/"shutdown" ping. + main_ping = self.wait_for_ping(self.restart_browser, MAIN_SHUTDOWN_PING) + + # Ensure the "main" ping has changed its client id. + self.assertIn("clientId", main_ping) + self.assertIsValidUUID(main_ping["clientId"]) + self.assertNotEqual(main_ping["clientId"], client_id) + + # Ensure we note in the ping that the user opted in. + parent_scalars = main_ping["payload"]["processes"]["parent"]["scalars"] + + self.assertIn("telemetry.data_upload_optin", parent_scalars) + self.assertIs(parent_scalars["telemetry.data_upload_optin"], True) + + # Ensure all pings sent during this test don't have the c0ffee client id. + for ping in self.ping_server.pings: + if "clientId" in ping: + self.assertIsValidUUID(ping["clientId"]) diff --git a/toolkit/components/telemetry/tests/marionette/tests/client/test_event_ping.py b/toolkit/components/telemetry/tests/marionette/tests/client/test_event_ping.py new file mode 100644 index 0000000000..97062642c0 --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/tests/client/test_event_ping.py @@ -0,0 +1,68 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import textwrap + +from telemetry_harness.testcase import TelemetryTestCase +from telemetry_harness.ping_filters import EVENT_PING + + +class TestEventPing(TelemetryTestCase): + """Tests for "event" ping.""" + + def enable_search_events(self): + """ + Event Telemetry categories are disabled by default. + Search events are in the "navigation" category and are not enabled by + default in builds of Firefox, so we enable them here. + """ + + script = """\ + let {Services} = ChromeUtils.import("resource://gre/modules/Services.jsm"); + Services.telemetry.setEventRecordingEnabled("navigation", true); + """ + + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + self.marionette.execute_script(textwrap.dedent(script)) + + def wait_for_search_service_init(self): + script = """\ + let [resolve] = arguments; + let searchService = Components.classes["@mozilla.org/browser/search-service;1"] + .getService(Components.interfaces.nsISearchService); + searchService.init().then(resolve); + """ + + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + self.marionette.execute_async_script(textwrap.dedent(script)) + + def test_event_ping(self): + """ + Barebones test for "event" ping: + Search, close Firefox, check "event" ping for search events. + """ + + self.enable_search_events() + self.wait_for_search_service_init() + + self.search("mozilla firefox") + + payload = self.wait_for_ping(self.restart_browser, EVENT_PING)["payload"] + + self.assertEqual(payload["reason"], "shutdown") + self.assertEqual(payload["lostEventsCount"], 0) + + self.assertIn("events", payload) + self.assertIn("parent", payload["events"]) + found_it = False + + for event in payload["events"]["parent"]: + # The event may optionally contain additonal fields + [timestamp, category, method, obj] = event[:4] + + self.assertTrue(timestamp > 0) + if category == "navigation" and method == "search" and obj == "urlbar": + found_it = True + + self.assertTrue(found_it) diff --git a/toolkit/components/telemetry/tests/marionette/tests/client/test_fog_custom_ping.py b/toolkit/components/telemetry/tests/marionette/tests/client/test_fog_custom_ping.py new file mode 100644 index 0000000000..11bf07472c --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/tests/client/test_fog_custom_ping.py @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from telemetry_harness.fog_ping_filters import FOG_ONE_PING_ONLY_PING +from telemetry_harness.fog_testcase import FOGTestCase + + +class TestDeletionRequestPing(FOGTestCase): + """Tests for the "one-ping-only" FOG custom ping.""" + + def test_one_ping_only_ping(self): + def send_opo_ping(marionette): + ping_sending_script = "GleanPings.onePingOnly.submit();" + with marionette.using_context(marionette.CONTEXT_CHROME): + marionette.execute_script(ping_sending_script) + + ping1 = self.wait_for_ping( + lambda: send_opo_ping(self.marionette), + FOG_ONE_PING_ONLY_PING, + ping_server=self.fog_ping_server, + ) + + self.assertNotIn("client_id", ping1["payload"]["client_info"]) diff --git a/toolkit/components/telemetry/tests/marionette/tests/client/test_fog_deletion_request_ping.py b/toolkit/components/telemetry/tests/marionette/tests/client/test_fog_deletion_request_ping.py new file mode 100644 index 0000000000..c273bc52be --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/tests/client/test_fog_deletion_request_ping.py @@ -0,0 +1,65 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import textwrap + +from telemetry_harness.fog_ping_filters import FOG_DELETION_REQUEST_PING +from telemetry_harness.fog_testcase import FOGTestCase + + +class TestDeletionRequestPing(FOGTestCase): + """Tests for FOG deletion-request ping.""" + + def test_deletion_request_ping_across_sessions(self): + """Test the "deletion-request" ping behaviour across sessions.""" + + self.search_in_new_tab("mozilla firefox") + + ping1 = self.wait_for_ping( + self.disable_telemetry, + FOG_DELETION_REQUEST_PING, + ping_server=self.fog_ping_server, + ) + + self.assertIn("ping_info", ping1["payload"]) + self.assertIn("client_info", ping1["payload"]) + + self.assertIn("client_id", ping1["payload"]["client_info"]) + client_id1 = ping1["payload"]["client_info"]["client_id"] + self.assertIsValidUUID(client_id1) + + self.restart_browser() + + self.assertEqual(self.fog_ping_server.pings[-1], ping1) + + self.enable_telemetry() + self.restart_browser() + + debug_tag = "my-test-tag" + tagging_script = """\ + let FOG = Components.classes["@mozilla.org/toolkit/glean;1"] + .createInstance(Components.interfaces.nsIFOG); + FOG.setTagPings("{}"); + """.format( + debug_tag + ) + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + self.marionette.execute_script(textwrap.dedent(tagging_script)) + self.search_in_new_tab("python unittest") + + ping2 = self.wait_for_ping( + self.disable_telemetry, + FOG_DELETION_REQUEST_PING, + ping_server=self.fog_ping_server, + ) + + self.assertEqual(ping2["debug_tag"], debug_tag) + + self.assertIn("client_id", ping2["payload"]["client_info"]) + client_id2 = ping2["payload"]["client_info"]["client_id"] + self.assertIsValidUUID(client_id2) + + # Verify that FOG creates a new client ID when a user + # opts out of sending technical and interaction data. + self.assertNotEqual(client_id2, client_id1) diff --git a/toolkit/components/telemetry/tests/marionette/tests/client/test_main_tab_scalars.py b/toolkit/components/telemetry/tests/marionette/tests/client/test_main_tab_scalars.py new file mode 100644 index 0000000000..8c9102b932 --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/tests/client/test_main_tab_scalars.py @@ -0,0 +1,39 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from telemetry_harness.testcase import TelemetryTestCase +from telemetry_harness.ping_filters import MAIN_SHUTDOWN_PING + + +class TestMainTabScalars(TelemetryTestCase): + """Tests for Telemetry Scalars.""" + + def test_main_tab_scalars(self): + """Test for Telemetry Scalars.""" + + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + start_tab = self.marionette.current_window_handle + + tab2 = self.open_tab(focus=True) + self.marionette.switch_to_window(tab2) + + tab3 = self.open_tab(focus=True) + self.marionette.switch_to_window(tab3) + + self.marionette.close() + self.marionette.switch_to_window(tab2) + + self.marionette.close() + self.marionette.switch_to_window(start_tab) + + ping = self.wait_for_ping(self.restart_browser, MAIN_SHUTDOWN_PING) + + self.assertEqual(ping["type"], "main") + self.assertEqual(ping["clientId"], self.client_id) + + scalars = ping["payload"]["processes"]["parent"]["scalars"] + + self.assertEqual(scalars["browser.engagement.max_concurrent_tab_count"], 3) + self.assertEqual(scalars["browser.engagement.tab_open_event_count"], 2) + self.assertEqual(scalars["browser.engagement.max_concurrent_window_count"], 1) diff --git a/toolkit/components/telemetry/tests/marionette/tests/client/test_search_counts_across_sessions.py b/toolkit/components/telemetry/tests/marionette/tests/client/test_search_counts_across_sessions.py new file mode 100644 index 0000000000..1ae7378891 --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/tests/client/test_search_counts_across_sessions.py @@ -0,0 +1,212 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import textwrap + +from telemetry_harness.testcase import TelemetryTestCase +from telemetry_harness.ping_filters import ( + MAIN_ENVIRONMENT_CHANGE_PING, + MAIN_SHUTDOWN_PING, +) + + +class TestSearchCounts(TelemetryTestCase): + """Test for SEARCH_COUNTS across sessions.""" + + def get_default_search_engine(self): + """Retrieve the identifier of the default search engine. + + We found that it's required to initialize the search service before + attempting to retrieve the default search engine. Not calling init + would result in a JavaScript error (see bug 1543960 for more + information). + """ + + script = """\ + let [resolve] = arguments; + let searchService = Components.classes[ + "@mozilla.org/browser/search-service;1"] + .getService(Components.interfaces.nsISearchService); + return searchService.init().then(function () { + resolve(searchService.defaultEngine.identifier); + }); + """ + + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + return self.marionette.execute_async_script(textwrap.dedent(script)) + + def setUp(self): + """Set up the test case and store the identifier of the default + search engine, which is required for reading SEARCH_COUNTS from + keyed histograms in pings. + """ + super(TestSearchCounts, self).setUp() + self.search_engine = self.get_default_search_engine() + + def test_search_counts(self): + """Test for SEARCH_COUNTS across sessions.""" + + # Session S1, subsession 1: + # - Open browser + # - Open new tab + # - Perform search (awesome bar or search bar) + # - Restart browser in new session + + self.search_in_new_tab("mozilla firefox") + + ping1 = self.wait_for_ping(self.restart_browser, MAIN_SHUTDOWN_PING) + + # Session S2, subsession 1: + # - Outcome 1 + # - Received a main ping P1 for previous session + # - Ping base contents: + # - clientId should be set + # - sessionId should be set + # - subsessionId should be set + # - previousSessionId should not be set + # - previousSubsessionId should not be set + # - subSessionCounter should be 1 + # - profileSubSessionCounter should be 1 + # - reason should be "shutdown" + # - Other ping contents: + # - SEARCH_COUNTS values should match performed search action + + client_id = ping1["clientId"] + self.assertIsValidUUID(client_id) + + ping1_info = ping1["payload"]["info"] + self.assertEqual(ping1_info["reason"], "shutdown") + + s1_session_id = ping1_info["sessionId"] + self.assertNotEqual(s1_session_id, "") + + s1_s1_subsession_id = ping1_info["subsessionId"] + self.assertNotEqual(s1_s1_subsession_id, "") + + self.assertIsNone(ping1_info["previousSessionId"]) + self.assertIsNone(ping1_info["previousSubsessionId"]) + self.assertEqual(ping1_info["subsessionCounter"], 1) + self.assertEqual(ping1_info["profileSubsessionCounter"], 1) + + scalars1 = ping1["payload"]["processes"]["parent"]["scalars"] + self.assertNotIn("browser.engagement.window_open_event_count", scalars1) + self.assertEqual(scalars1["browser.engagement.tab_open_event_count"], 1) + + keyed_histograms1 = ping1["payload"]["keyedHistograms"] + search_counts1 = keyed_histograms1["SEARCH_COUNTS"][ + "{}.urlbar".format(self.search_engine) + ] + self.assertEqual( + search_counts1, + { + u"range": [1, 2], + u"bucket_count": 3, + u"histogram_type": 4, + u"values": {u"1": 0, u"0": 1}, + u"sum": 1, + }, + ) + + # - Install addon + # Session S2, subsession 2: + # - Outcome 2 + # - Received a main ping P2 for previous subsession + # - Ping base contents: + # - clientId should be set to the same value + # - sessionId should be set to a new value + # - subsessionId should be set to a new value + # - previousSessionId should be set to P1s sessionId value + # - previousSubsessionId should be set to P1s subsessionId value + # - subSessionCounter should be 1 + # - profileSubSessionCounter should be 2 + # - reason should be "environment-change" + # - Other ping contents: + # - SEARCH_COUNTS values should not be in P2 + # - Verify that there should be no listing for tab scalar as we started a new + # session + + ping2 = self.wait_for_ping(self.install_addon, MAIN_ENVIRONMENT_CHANGE_PING) + + self.assertEqual(ping2["clientId"], client_id) + + ping2_info = ping2["payload"]["info"] + self.assertEqual(ping2_info["reason"], "environment-change") + + s2_session_id = ping2_info["sessionId"] + self.assertNotEqual(s2_session_id, s1_session_id) + + s2_s1_subsession_id = ping2_info["subsessionId"] + self.assertNotEqual(s2_s1_subsession_id, s1_s1_subsession_id) + + self.assertEqual(ping2_info["previousSessionId"], s1_session_id) + self.assertEqual(ping2_info["previousSubsessionId"], s1_s1_subsession_id) + self.assertEqual(ping2_info["subsessionCounter"], 1) + self.assertEqual(ping2_info["profileSubsessionCounter"], 2) + + scalars2 = ping2["payload"]["processes"]["parent"]["scalars"] + self.assertNotIn("browser.engagement.window_open_event_count", scalars2) + self.assertNotIn("browser.engagement.tab_open_event_count", scalars2) + + keyed_histograms2 = ping2["payload"]["keyedHistograms"] + self.assertNotIn("SEARCH_COUNTS", keyed_histograms2) + + # - Perform Search + # - Restart Browser + + self.search("mozilla telemetry") + self.search("python unittest") + self.search("python pytest") + + ping3 = self.wait_for_ping(self.restart_browser, MAIN_SHUTDOWN_PING) + + # Session S3, subsession 1: + # - Outcome 3 + # - Received a main ping P3 for session 2, subsession 1 + # - Ping base contents: + # - clientId should be set to the same value + # - sessionId should be set to P2s sessionId value + # - subsessionId should be set to a new value + # - previousSessionId should be set to P1s sessionId value + # - previousSubsessionId should be set to P2s subsessionId value + # - subSessionCounter should be 2 + # - profileSubSessionCounter should be 3 + # - reason should be "shutdown" + # - Other ping contents: + # - SEARCH_COUNTS values should be set per above search + + self.assertEqual(ping3["clientId"], client_id) + + ping3_info = ping3["payload"]["info"] + + self.assertEqual(ping3_info["reason"], "shutdown") + + self.assertEqual(ping3_info["sessionId"], s2_session_id) + + s2_s2_subsession_id = ping3_info["subsessionId"] + self.assertNotEqual(s2_s2_subsession_id, s1_s1_subsession_id) + self.assertNotEqual(s2_s2_subsession_id, s2_s1_subsession_id) + + self.assertEqual(ping3_info["previousSessionId"], s1_session_id) + self.assertEqual(ping3_info["previousSubsessionId"], s2_s1_subsession_id) + self.assertEqual(ping3_info["subsessionCounter"], 2) + self.assertEqual(ping3_info["profileSubsessionCounter"], 3) + + scalars3 = ping3["payload"]["processes"]["parent"]["scalars"] + self.assertNotIn("browser.engagement.window_open_event_count", scalars3) + self.assertNotIn("browser.engagement.tab_open_event_count", scalars3) + + keyed_histograms3 = ping3["payload"]["keyedHistograms"] + search_counts3 = keyed_histograms3["SEARCH_COUNTS"][ + "{}.urlbar".format(self.search_engine) + ] + self.assertEqual( + search_counts3, + { + u"range": [1, 2], + u"bucket_count": 3, + u"histogram_type": 4, + u"values": {u"1": 0, u"0": 3}, + u"sum": 3, + }, + ) diff --git a/toolkit/components/telemetry/tests/marionette/tests/client/test_subsession_management.py b/toolkit/components/telemetry/tests/marionette/tests/client/test_subsession_management.py new file mode 100644 index 0000000000..79b7835e5c --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/tests/client/test_subsession_management.py @@ -0,0 +1,147 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from telemetry_harness.testcase import TelemetryTestCase +from telemetry_harness.ping_filters import ( + MAIN_ENVIRONMENT_CHANGE_PING, + MAIN_SHUTDOWN_PING, +) + + +class TestSubsessionManagement(TelemetryTestCase): + """Tests for Firefox Telemetry subsession management.""" + + def test_subsession_management(self): + """Test for Firefox Telemetry subsession management.""" + + # Session S1, subsession 1 + # Actions: + # 1. Open browser + # 2. Open a new tab + # 3. Restart browser in new session + + with self.new_tab(): + # If Firefox Telemetry is working correctly, this will + # be sufficient to record a tab open event. + pass + + ping1 = self.wait_for_ping(self.restart_browser, MAIN_SHUTDOWN_PING) + + # Session S2, subsession 1 + # Outcome 1: + # Received a main ping P1 for previous session + # - Ping base contents: + # - clientId should be a valid UUID + # - reason should be "shutdown" + # - sessionId should be set + # - subsessionId should be set + # - previousSessionId should not be set + # - previousSubsessionId should not be set + # - subSessionCounter should be 1 + # - profileSubSessionCounter should be 1 + # - Other ping contents: + # - tab_open_event_count in scalars + + client_id = ping1["clientId"] + self.assertIsValidUUID(client_id) + + ping1_info = ping1["payload"]["info"] + self.assertEqual(ping1_info["reason"], "shutdown") + + s1_session_id = ping1_info["sessionId"] + self.assertNotEqual(s1_session_id, "") + + s1_s1_subsession_id = ping1_info["subsessionId"] + self.assertNotEqual(s1_s1_subsession_id, "") + self.assertIsNone(ping1_info["previousSessionId"]) + self.assertIsNone(ping1_info["previousSubsessionId"]) + self.assertEqual(ping1_info["subsessionCounter"], 1) + self.assertEqual(ping1_info["profileSubsessionCounter"], 1) + + scalars1 = ping1["payload"]["processes"]["parent"]["scalars"] + self.assertNotIn("browser.engagement.window_open_event_count", scalars1) + self.assertEqual(scalars1["browser.engagement.tab_open_event_count"], 1) + + # Actions: + # 1. Install addon + + ping2 = self.wait_for_ping(self.install_addon, MAIN_ENVIRONMENT_CHANGE_PING) + + [addon_id] = self.addon_ids # Store the addon ID for verifying ping3 later + + # Session S2, subsession 2 + # Outcome 2: + # Received a main ping P2 for previous subsession + # - Ping base contents: + # - clientId should be set to the same value + # - sessionId should be set to a new value + # - subsessionId should be set to a new value + # - previousSessionId should be set to P1s sessionId value + # - previousSubsessionId should be set to P1s subsessionId value + # - subSessionCounter should be 1 + # - profileSubSessionCounter should be 2 + # - reason should be "environment-change" + # - Other ping contents: + # - tab_open_event_count not in scalars + + self.assertEqual(ping2["clientId"], client_id) + + ping2_info = ping2["payload"]["info"] + self.assertEqual(ping2_info["reason"], "environment-change") + + s2_session_id = ping2_info["sessionId"] + self.assertNotEqual(s2_session_id, s1_session_id) + + s2_s1_subsession_id = ping2_info["subsessionId"] + self.assertNotEqual(s2_s1_subsession_id, s1_s1_subsession_id) + self.assertEqual(ping2_info["previousSessionId"], s1_session_id) + self.assertEqual(ping2_info["previousSubsessionId"], s1_s1_subsession_id) + self.assertEqual(ping2_info["subsessionCounter"], 1) + self.assertEqual(ping2_info["profileSubsessionCounter"], 2) + + scalars2 = ping2["payload"]["processes"]["parent"]["scalars"] + self.assertNotIn("browser.engagement.window_open_event_count", scalars2) + self.assertNotIn("browser.engagement.tab_open_event_count", scalars2) + + # Actions + # 1. Restart browser in new session + + ping3 = self.wait_for_ping(self.restart_browser, MAIN_SHUTDOWN_PING) + + # Session S3, subsession 1 + # Outcome 3: + # Received a main ping P3 for session 2, subsession 2 + # - Ping base contents: + # - clientId should be set to the same value + # - sessionId should be set to P2s sessionId value + # - subsessionId should be set to a new value + # - previousSessionId should be set to P1s sessionId value + # - previousSubsessionId should be set to P2s subsessionId value + # - subSessionCounter should be 2 + # - profileSubSessionCounter should be 3 + # - reason should be "shutdown" + # - Other ping contents: + # - addon ID in activeAddons in environment + + self.assertEqual(ping3["clientId"], client_id) + + ping3_info = ping3["payload"]["info"] + self.assertEqual(ping3_info["reason"], "shutdown") + + self.assertEqual(ping3_info["sessionId"], s2_session_id) + + s2_s2_subsession_id = ping3_info["subsessionId"] + self.assertNotEqual(s2_s2_subsession_id, s1_s1_subsession_id) + self.assertNotEqual(s2_s2_subsession_id, s2_s1_subsession_id) + self.assertEqual(ping3_info["previousSessionId"], s1_session_id) + self.assertEqual(ping3_info["previousSubsessionId"], s2_s1_subsession_id) + self.assertEqual(ping3_info["subsessionCounter"], 2) + self.assertEqual(ping3_info["profileSubsessionCounter"], 3) + + scalars3 = ping3["payload"]["processes"]["parent"]["scalars"] + self.assertNotIn("browser.engagement.window_open_event_count", scalars3) + self.assertNotIn("browser.engagement.tab_open_event_count", scalars3) + + active_addons = ping3["environment"]["addons"]["activeAddons"] + self.assertIn(addon_id, active_addons) diff --git a/toolkit/components/telemetry/tests/marionette/tests/manifest.ini b/toolkit/components/telemetry/tests/marionette/tests/manifest.ini new file mode 100644 index 0000000000..b5e6f442e9 --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/tests/manifest.ini @@ -0,0 +1,2 @@ +[include:unit/manifest.ini] +[include:client/manifest.ini] diff --git a/toolkit/components/telemetry/tests/marionette/tests/unit/manifest.ini b/toolkit/components/telemetry/tests/marionette/tests/unit/manifest.ini new file mode 100644 index 0000000000..9bb2de707a --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/tests/unit/manifest.ini @@ -0,0 +1,4 @@ +[DEFAULT] +tags = unit + +[test_ping_server_received_ping.py]
\ No newline at end of file diff --git a/toolkit/components/telemetry/tests/marionette/tests/unit/test_ping_server_received_ping.py b/toolkit/components/telemetry/tests/marionette/tests/unit/test_ping_server_received_ping.py new file mode 100644 index 0000000000..71da6bef8f --- /dev/null +++ b/toolkit/components/telemetry/tests/marionette/tests/unit/test_ping_server_received_ping.py @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/ + +import requests + +from telemetry_harness.testcase import TelemetryTestCase + + +class TestPingServer(TelemetryTestCase): + def setUp(self, *args, **kwargs): + """Set up the test case retrieve the pings URL.""" + super(TestPingServer, self).setUp(*args, **kwargs) + self.pings_url = self.ping_server.get_url("/pings") + + def test_ping_server_received_ping(self): + ping_type = "server-test-ping" + ping_reason = "unit-test" + + def send_ping_request(): + """Perform a POST request to the ping server.""" + data = {"type": ping_type, "reason": ping_reason} + headers = { + "Content-type": "application/json", + "Accept": "text/plain", + } + + response = requests.post(self.pings_url, json=data, headers=headers) + + self.assertEqual( + response.status_code, + 200, + msg="Error sending POST request to ping server: {response.text}".format( + response=response + ), + ) + return response + + def ping_filter_func(ping): + return ping["type"] == ping_type + + [ping] = self.wait_for_pings(send_ping_request, ping_filter_func, 1) + + self.assertIsNotNone(ping) + self.assertEqual(ping["type"], ping_type) + self.assertEqual(ping["reason"], ping_reason) |