summaryrefslogtreecommitdiffstats
path: root/python/mozperftest/mozperftest/tests/test_webpagetest.py
diff options
context:
space:
mode:
Diffstat (limited to 'python/mozperftest/mozperftest/tests/test_webpagetest.py')
-rw-r--r--python/mozperftest/mozperftest/tests/test_webpagetest.py271
1 files changed, 271 insertions, 0 deletions
diff --git a/python/mozperftest/mozperftest/tests/test_webpagetest.py b/python/mozperftest/mozperftest/tests/test_webpagetest.py
new file mode 100644
index 0000000000..1c2c51ca0b
--- /dev/null
+++ b/python/mozperftest/mozperftest/tests/test_webpagetest.py
@@ -0,0 +1,271 @@
+import json
+import random
+from unittest import mock
+
+import pytest
+import requests
+
+import mozperftest.test.webpagetest as webpagetest
+from mozperftest.test.webpagetest import (
+ ACCEPTED_STATISTICS,
+ WPT_API_EXPIRED_MESSAGE,
+ WPTBrowserSelectionError,
+ WPTDataProcessingError,
+ WPTExpiredAPIKeyError,
+ WPTInvalidConnectionSelection,
+ WPTInvalidStatisticsError,
+ WPTInvalidURLError,
+ WPTLocationSelectionError,
+ WPTTimeOutError,
+)
+from mozperftest.tests.support import EXAMPLE_WPT_TEST, get_running_env
+
+WPT_METRICS = [
+ "firstContentfulPaint",
+ "timeToContentfulPaint",
+ "visualComplete90",
+ "firstPaint",
+ "visualComplete99",
+ "visualComplete",
+ "SpeedIndex",
+ "bytesIn",
+ "bytesOut",
+ "TTFB",
+ "fullyLoadedCPUms",
+ "fullyLoadedCPUpct",
+ "domElements",
+ "domContentLoadedEventStart",
+ "domContentLoadedEventEnd",
+ "loadEventStart",
+ "loadEventEnd",
+]
+
+
+class WPTTests:
+ def __init__(self, log):
+ self.log = log
+
+ def runTests(self, args):
+ return True
+
+
+def running_env(**kw):
+ return get_running_env(flavor="webpagetest", **kw)
+
+
+def init_placeholder_wpt_data(fvonly=False, invalid_results=False):
+ views = {"firstView": {}}
+ if not fvonly:
+ views["repeatView"] = {}
+ placeholder_data = {
+ "data": {
+ "summary": "websitelink.com",
+ "location": "ec2-us-east-1:Firefox",
+ "testRuns": 3,
+ "successfulFVRuns": 3,
+ "successfulRVRuns": 3,
+ "fvonly": fvonly,
+ "average": views,
+ "standardDeviation": views,
+ "median": views,
+ "runs": {"1": {"firstView": {"browserVersion": 101.1}}},
+ "url": "testurl.ca",
+ },
+ "webPagetestVersion": 21.0,
+ }
+ exclude_metrics = 0 if not invalid_results else 2
+ for metric in WPT_METRICS[exclude_metrics:]:
+ for view in views:
+ for stat in ACCEPTED_STATISTICS:
+ placeholder_data["data"][stat][view][metric] = random.randint(0, 10000)
+ placeholder_data["data"][stat][view][metric] = random.randint(0, 10000)
+ placeholder_data["data"][stat][view][metric] = random.randint(0, 10000)
+ return placeholder_data
+
+
+def init_mocked_request(
+ status_code, WPT_test_status_code=200, WPT_test_status_text="Ok", **kwargs
+):
+ mock_data = {
+ "data": {
+ "ec2-us-east-1": {"PendingTests": {"Queued": 3}, "Label": "California"},
+ "jsonUrl": "mock_test.com",
+ "summary": "Just a pageload test",
+ "url": "testurl.ca",
+ "remaining": 2000,
+ },
+ "statusCode": WPT_test_status_code,
+ "statusText": WPT_test_status_text,
+ }
+ for key, value in kwargs.items():
+ mock_data["data"][key] = value
+ mock_request = requests.Response()
+ mock_request.status_code = status_code
+ mock_request._content = json.dumps(mock_data).encode("utf-8")
+ return mock_request
+
+
+@mock.patch(
+ "mozperftest.test.webpagetest.WebPageTest.get_WPT_results",
+ return_value=init_placeholder_wpt_data(),
+)
+@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
+@mock.patch(
+ "mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
+)
+@mock.patch("requests.get", return_value=init_mocked_request(200))
+@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
+def test_webpagetest_no_issues_mocked_results(*mocked):
+ mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
+ test = webpagetest.WebPageTest(env, mach_cmd)
+ metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1
+ metadata.script["options"]["test_parameters"]["first_view_only"] = 0
+ metadata.script["options"]["test_parameters"]["test_list"] = ["google.ca"]
+ test.run(metadata)
+
+
+@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
+@mock.patch(
+ "mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
+)
+@mock.patch("requests.get", return_value=init_mocked_request(200))
+@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
+def test_webpagetest_test_invalid_browser(*mocked):
+ mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
+ metadata.script["options"]["test_parameters"]["browser"] = "Invalid Browser"
+ test = webpagetest.WebPageTest(env, mach_cmd)
+ with pytest.raises(WPTBrowserSelectionError):
+ test.run(metadata)
+
+
+@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
+@mock.patch(
+ "mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
+)
+@mock.patch("requests.get", return_value=init_mocked_request(200))
+@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
+def test_webpagetest_test_invalid_connection(*mocked):
+ mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
+ test = webpagetest.WebPageTest(env, mach_cmd)
+ metadata.script["options"]["test_parameters"]["connection"] = "Invalid Connection"
+ with pytest.raises(WPTInvalidConnectionSelection):
+ test.run(metadata)
+
+
+@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
+@mock.patch(
+ "mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
+)
+@mock.patch("requests.get", return_value=init_mocked_request(200))
+@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
+def test_webpagetest_test_invalid_url(*mocked):
+ mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
+ test = webpagetest.WebPageTest(env, mach_cmd)
+ metadata.script["options"]["test_list"] = ["InvalidUrl"]
+ with pytest.raises(WPTInvalidURLError):
+ test.run(metadata)
+
+
+@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
+@mock.patch(
+ "mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
+)
+@mock.patch("requests.get", return_value=init_mocked_request(200))
+@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
+def test_webpagetest_test_invalid_statistic(*mocked):
+ mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
+ test = webpagetest.WebPageTest(env, mach_cmd)
+ metadata.script["options"]["test_parameters"]["statistics"] = ["Invalid Statistic"]
+ with pytest.raises(WPTInvalidStatisticsError):
+ test.run(metadata)
+ assert True
+
+
+@mock.patch("requests.get", return_value=init_mocked_request(200))
+@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
+@mock.patch(
+ "mozperftest.test.webpagetest.WebPageTest.request_with_timeout",
+ return_value={"data": {}},
+)
+@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
+def test_webpagetest_test_invalid_location(*mocked):
+ mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
+ test = webpagetest.WebPageTest(env, mach_cmd)
+ metadata.script["options"]["test_parameters"]["location"] = "Invalid Location"
+ with pytest.raises(WPTLocationSelectionError):
+ test.run(metadata)
+
+
+@mock.patch("requests.get", return_value=init_mocked_request(200, 101))
+@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
+@mock.patch(
+ "mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
+)
+@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
+def test_webpagetest_test_timeout(*mocked):
+ mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
+ test = webpagetest.WebPageTest(env, mach_cmd)
+ metadata.script["options"]["test_parameters"]["timeout_limit"] = 2
+ metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1
+ with pytest.raises(WPTTimeOutError):
+ test.run(metadata)
+ assert True
+
+
+@mock.patch(
+ "requests.get",
+ return_value=init_mocked_request(
+ 200, testRuns=3, successfulFVRuns=3, fvonly=True, location="BadLocation"
+ ),
+)
+@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
+@mock.patch(
+ "mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
+)
+@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
+def test_webpagetest_test_wrong_browserlocation(*mocked):
+ mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
+ metadata.script["options"]["test_list"] = ["google.ca"]
+ metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1
+ test = webpagetest.WebPageTest(env, mach_cmd)
+ with pytest.raises(WPTBrowserSelectionError):
+ test.run(metadata)
+
+
+@mock.patch(
+ "mozperftest.test.webpagetest.WebPageTest.get_WPT_results",
+ return_value=init_placeholder_wpt_data(invalid_results=True),
+)
+@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
+@mock.patch(
+ "mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
+)
+@mock.patch("requests.get", return_value=init_mocked_request(200))
+@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
+def test_webpagetest_test_metric_not_found(*mocked):
+ mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
+ metadata.script["options"]["test_list"] = ["google.ca"]
+ metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1
+ test = webpagetest.WebPageTest(env, mach_cmd)
+ with pytest.raises(WPTDataProcessingError):
+ test.run(metadata)
+
+
+@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
+@mock.patch(
+ "mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
+)
+@mock.patch(
+ "requests.get",
+ return_value=init_mocked_request(
+ 200, WPT_test_status_code=400, WPT_test_status_text=WPT_API_EXPIRED_MESSAGE
+ ),
+)
+@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
+def test_webpagetest_test_expired_api_key(*mocked):
+ mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
+ metadata.script["options"]["test_list"] = ["google.ca"]
+ metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1
+ test = webpagetest.WebPageTest(env, mach_cmd)
+ with pytest.raises(WPTExpiredAPIKeyError):
+ test.run(metadata)
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-01 16:24:32 +0000