diff options
Diffstat (limited to 'python/mozperftest/mozperftest/tests/test_browsertime.py')
-rw-r--r-- | python/mozperftest/mozperftest/tests/test_browsertime.py | 364 |
1 files changed, 364 insertions, 0 deletions
diff --git a/python/mozperftest/mozperftest/tests/test_browsertime.py b/python/mozperftest/mozperftest/tests/test_browsertime.py new file mode 100644 index 0000000000..e0cd298da7 --- /dev/null +++ b/python/mozperftest/mozperftest/tests/test_browsertime.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python +import os +import pathlib +import random +import shutil +import string +from unittest import mock + +import mozunit +import pytest + +from mozperftest.environment import SYSTEM, TEST +from mozperftest.test.browsertime import add_options +from mozperftest.test.browsertime.runner import ( + NodeException, + extract_browser_name, + matches, +) +from mozperftest.tests.support import EXAMPLE_TEST, get_running_env +from mozperftest.utils import silence, temporary_env + +HERE = os.path.dirname(__file__) + +# Combine these dictionaries as required for mocking the +# Browsertime installation related methods +BTIME_PKG_DEP = { + "devDependencies": {"browsertime": "89771a1d6be54114db190427dbc281582cba3d47"} +} +BTIME_PKG_NO_INSTALL = { + "packages": { + "node_modules/browsertime": { + "resolved": ( + "browsertime@https://github.com/sitespeedio/browsertime" + "/tarball/89771a1d6be54114db190427dbc281582cba3d47" + ) + } + } +} +BTIME_PKG_REINSTALL = { + "packages": { + "node_modules/browsertime": { + "resolved": ( + "browsertime@https://github.com/sitespeedio/browsertime" + "/tarball/98747854be54114db190427dbc281582cba3d47" + ) + } + } +} + + +def fetch(self, url): + return os.path.join(HERE, "fetched_artifact.zip") + + +def mocked_jsonload(val): + return val.__iter__.return_value + + +def build_mock_open(files_data): + mocked_opens = [] + + for data in files_data: + mocked_file = mock.MagicMock() + mocked_file.__enter__.return_value.__iter__.return_value = data + mocked_opens.append(mocked_file) + + m = mock.mock_open() + m.side_effect = mocked_opens + return m + + +@mock.patch("mozperftest.test.browsertime.runner.install_package") +@mock.patch( + "mozperftest.test.noderunner.NodeRunner.verify_node_install", new=lambda x: True +) +@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch) +@mock.patch( + "mozperftest.test.browsertime.runner.BrowsertimeRunner._setup_node_packages", + new=lambda x, y: None, +) +def test_browser(*mocked): + mach_cmd, metadata, env = get_running_env( + android=True, + android_app_name="something", + browsertime_geckodriver="GECKODRIVER", + browsertime_iterations=1, + browsertime_extra_options="one=1,two=2", + tests=[EXAMPLE_TEST], + browsertime_no_window_recorder=False, + browsertime_viewport_size="1234x567", + ) + + sys = env.layers[SYSTEM] + browser = env.layers[TEST] + try: + with sys as s, browser as b, silence(): + b(s(metadata)) + finally: + shutil.rmtree(mach_cmd._mach_context.state_dir) + assert mach_cmd.run_process.call_count == 1 + + # Make sure all arguments are of type str + for option in mach_cmd.run_process.call_args[0][0]: + assert isinstance(option, str) + + cmd = " ".join(mach_cmd.run_process.call_args[0][0]) + assert EXAMPLE_TEST in cmd + assert "--firefox.geckodriverPath GECKODRIVER" in cmd + assert "--one 1" in cmd + assert "--two 2" in cmd + + results = metadata.get_results() + assert len(results) == 1 + assert set(list(results[0].keys())) - set(["name", "results"]) == set() + assert results[0]["name"] == "Example" + + +@mock.patch( + "mozperftest.test.browsertime.runner.BrowsertimeRunner.browsertime_js", + new=pathlib.Path("doesn't-exist"), +) +@mock.patch( + "mozperftest.test.browsertime.runner.BrowsertimeRunner.visualmetrics_py", + new=pathlib.Path("doesn't-exist-either"), +) +def test_browsertime_not_existing(): + _, _, env = get_running_env( + android=True, + android_app_name="something", + browsertime_geckodriver="GECKODRIVER", + browsertime_iterations=1, + browsertime_extra_options="one=1,two=2", + tests=[EXAMPLE_TEST], + ) + browser = env.layers[TEST] + btime_layer = browser.layers[0] + assert btime_layer._should_install() + + +@mock.patch( + "mozperftest.test.browsertime.runner.pathlib.Path.exists", new=lambda x: True +) +def test_browsertime_no_reinstall(): + _, _, env = get_running_env( + android=True, + android_app_name="something", + browsertime_geckodriver="GECKODRIVER", + browsertime_iterations=1, + browsertime_extra_options="one=1,two=2", + tests=[EXAMPLE_TEST], + ) + + with mock.patch( + "mozperftest.test.browsertime.runner.pathlib.Path.open", + build_mock_open([BTIME_PKG_DEP, BTIME_PKG_NO_INSTALL]), + ), mock.patch("mozperftest.test.browsertime.runner.json.load", new=mocked_jsonload): + browser = env.layers[TEST] + btime_layer = browser.layers[0] + assert not btime_layer._should_install() + + +@mock.patch( + "mozperftest.test.browsertime.runner.pathlib.Path.exists", new=lambda x: True +) +def test_browsertime_should_reinstall(): + _, _, env = get_running_env( + android=True, + android_app_name="something", + browsertime_geckodriver="GECKODRIVER", + browsertime_iterations=1, + browsertime_extra_options="one=1,two=2", + tests=[EXAMPLE_TEST], + ) + + with mock.patch( + "mozperftest.test.browsertime.runner.pathlib.Path.open", + build_mock_open([BTIME_PKG_DEP, BTIME_PKG_REINSTALL]), + ), mock.patch("mozperftest.test.browsertime.runner.json.load", new=mocked_jsonload): + browser = env.layers[TEST] + btime_layer = browser.layers[0] + assert btime_layer._should_install() + + +@mock.patch("mozperftest.test.browsertime.runner.install_package") +@mock.patch( + "mozperftest.test.noderunner.NodeRunner.verify_node_install", new=lambda x: True +) +@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch) +@mock.patch( + "mozperftest.test.browsertime.runner.BrowsertimeRunner._setup_node_packages", + new=lambda x, y: None, +) +def test_browser_failed(*mocked): + mach_cmd, metadata, env = get_running_env( + android=True, + android_app_name="something", + browsertime_geckodriver="GECKODRIVER", + browsertime_iterations=1, + browsertime_extra_options="one=1,two=2", + tests=[EXAMPLE_TEST], + browsertime_no_window_recorder=False, + browsertime_viewport_size="1234x567", + ) + # set the return value to 1 to simulate a node failure + mach_cmd.run_process.return_value = 1 + browser = env.layers[TEST] + sys = env.layers[SYSTEM] + + with sys as s, browser as b, silence(), pytest.raises(NodeException): + b(s(metadata)) + + +@mock.patch("mozperftest.test.browsertime.runner.install_package") +@mock.patch( + "mozperftest.test.noderunner.NodeRunner.verify_node_install", new=lambda x: True +) +@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch) +@mock.patch( + "mozperftest.test.browsertime.runner.BrowsertimeRunner._setup_node_packages", + new=lambda x, y: None, +) +def test_browser_desktop(*mocked): + mach_cmd, metadata, env = get_running_env( + browsertime_iterations=1, + browsertime_extra_options="one=1,two=2", + tests=[EXAMPLE_TEST], + browsertime_no_window_recorder=False, + browsertime_viewport_size="1234x567", + ) + browser = env.layers[TEST] + sys = env.layers[SYSTEM] + + try: + with sys as s, browser as b, silence(): + # just checking that the setup_helper property gets + # correctly initialized + browsertime = browser.layers[-1] + assert browsertime.setup_helper is not None + helper = browsertime.setup_helper + assert browsertime.setup_helper is helper + + b(s(metadata)) + finally: + shutil.rmtree(mach_cmd._mach_context.state_dir) + + assert mach_cmd.run_process.call_count == 1 + cmd = " ".join(mach_cmd.run_process.call_args[0][0]) + # check that --firefox.binaryPath is set automatically + assert "--firefox.binaryPath" in cmd + + +@mock.patch("mozperftest.test.browsertime.runner.install_package") +@mock.patch( + "mozperftest.test.noderunner.NodeRunner.verify_node_install", new=lambda x: True +) +@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch) +@mock.patch( + "mozperftest.test.browsertime.runner.BrowsertimeRunner._setup_node_packages", + new=lambda x, y: None, +) +def test_existing_results(*mocked): + mach_cmd, metadata, env = get_running_env( + browsertime_existing_results="/some/path", + tests=[EXAMPLE_TEST], + ) + browser = env.layers[TEST] + sys = env.layers[SYSTEM] + + try: + with sys as s, browser as b, silence(): + # just checking that the setup_helper property gets + # correctly initialized + browsertime = browser.layers[-1] + assert browsertime.setup_helper is not None + helper = browsertime.setup_helper + assert browsertime.setup_helper is helper + + m = b(s(metadata)) + results = m.get_results() + assert len(results) == 1 + assert results[0]["results"] == "/some/path" + assert results[0]["name"] == "Example" + finally: + shutil.rmtree(mach_cmd._mach_context.state_dir) + + assert mach_cmd.run_process.call_count == 0 + + +def test_add_options(): + mach_cmd, metadata, env = get_running_env() + options = [("one", 1), ("two", 2)] + add_options(env, options) + extra = env.get_arg("browsertime-extra-options") + assert "one=1" in extra + assert "two=2" in extra + + +@mock.patch("mozperftest.test.browsertime.runner.install_package") +@mock.patch( + "mozperftest.test.noderunner.NodeRunner.verify_node_install", new=lambda x: True +) +@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch) +@mock.patch("mozperftest.test.browsertime.runner.BrowsertimeRunner.setup_helper") +def test_install_url(*mocked): + url = "https://here/tarball/" + "".join( + [random.choice(string.hexdigits[:-6]) for c in range(40)] + ) + mach, metadata, env = get_running_env( + browsertime_install_url=url, + tests=[EXAMPLE_TEST], + browsertime_no_window_recorder=False, + browsertime_viewport_size="1234x567", + ) + browser = env.layers[TEST] + sys = env.layers[SYSTEM] + + try: + with sys as s, temporary_env(MOZ_AUTOMATION="1"), browser as b, silence(): + b(s(metadata)) + finally: + shutil.rmtree(mach._mach_context.state_dir) + + assert mach.run_process.call_count == 1 + + +@mock.patch("mozperftest.test.browsertime.runner.install_package") +@mock.patch( + "mozperftest.test.noderunner.NodeRunner.verify_node_install", new=lambda x: True +) +@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch) +@mock.patch( + "mozperftest.test.browsertime.runner.BrowsertimeRunner._setup_node_packages", + new=lambda x, y: None, +) +def test_install_url_bad(*mocked): + mach, metadata, env = get_running_env( + browsertime_install_url="meh", + tests=[EXAMPLE_TEST], + ) + browser = env.layers[TEST] + sys = env.layers[SYSTEM] + + with pytest.raises(ValueError): + try: + with sys as s, browser as b, silence(): + b(s(metadata)) + finally: + shutil.rmtree(mach._mach_context.state_dir) + + +def test_matches(): + args = ["arg1=1", "--arg2=value2"] + + assert matches(args, "arg1") + assert not matches(args, "arg3") + + +def test_extract_browser_name(): + args = ["arg1=1", "--arg2=value2", "--browser=me", "--zome"] + assert extract_browser_name(args) == "me" + + +if __name__ == "__main__": + mozunit.main() |