path: root/testing/raptor/test/test_raptor.py

import os
import sys
import threading
import time
import traceback
from unittest import mock
from unittest.mock import Mock

import mozunit
import pytest
from mozprofile import BaseProfile
from mozrunner.errors import RunnerNotStartedError
from six import reraise

# need this so the raptor unit tests can find output & filter classes
here = os.path.abspath(os.path.dirname(__file__))
raptor_dir = os.path.join(os.path.dirname(here), "raptor")
sys.path.insert(0, raptor_dir)


from browsertime import BrowsertimeAndroid, BrowsertimeDesktop
from webextension import (
    WebExtensionAndroid,
    WebExtensionDesktopChrome,
    WebExtensionFirefox,
)

DEFAULT_TIMEOUT = 125


class TestBrowserThread(threading.Thread):
    def __init__(self, raptor_instance, tests, names):
        super(TestBrowserThread, self).__init__()
        self.raptor_instance = raptor_instance
        self.tests = tests
        self.names = names
        self.exc = None

    def print_error(self):
        if self.exc is None:
            return
        type, value, tb = self.exc
        traceback.print_exception(type, value, tb, None, sys.stderr)

    def run(self):
        try:
            self.raptor_instance.run_tests(self.tests, self.names)
        except BaseException:
            self.exc = sys.exc_info()


# Perftest tests
@pytest.mark.parametrize(
    "perftest_class, app_name",
    [
        [WebExtensionFirefox, "firefox"],
        [WebExtensionDesktopChrome, "chrome"],
        [WebExtensionDesktopChrome, "chromium"],
        [WebExtensionAndroid, "geckoview"],
        [BrowsertimeDesktop, "firefox"],
        [BrowsertimeDesktop, "chrome"],
        [BrowsertimeDesktop, "chromium"],
        [BrowsertimeAndroid, "geckoview"],
    ],
)
def test_build_profile(options, perftest_class, app_name, get_prefs):
    options["app"] = app_name

    # We need to do the mock ourselves because of how the perftest_class
    # is being defined
    original_get = perftest_class.get_browser_meta
    perftest_class.get_browser_meta = mock.MagicMock()
    perftest_class.get_browser_meta.return_value = (app_name, "100")

    perftest_instance = perftest_class(**options)
    perftest_class.get_browser_meta = original_get

    assert isinstance(perftest_instance.profile, BaseProfile)
    if app_name != "firefox":
        return

    # These prefs are set in mozprofile
    firefox_prefs = [
        'user_pref("app.update.checkInstallTime", false);',
        'user_pref("app.update.disabledForTesting", true);',
        'user_pref("'
        'security.turn_off_all_security_so_that_viruses_can_take_over_this_computer", true);',
    ]
    # This pref is set in raptor
    raptor_pref = 'user_pref("security.enable_java", false);'

    prefs_file = os.path.join(perftest_instance.profile.profile, "user.js")
    with open(prefs_file, "r") as fh:
        prefs = fh.read()
        for firefox_pref in firefox_prefs:
            assert firefox_pref in prefs
        assert raptor_pref in prefs


def test_perftest_host_ip(ConcretePerftest, options, get_prefs):
    os.environ["HOST_IP"] = "some_dummy_host_ip"
    options["host"] = "HOST_IP"

    perftest = ConcretePerftest(**options)

    assert perftest.config["host"] == os.environ["HOST_IP"]


@pytest.mark.parametrize(
    "app_name, expected_e10s_flag",
    [["firefox", True], ["geckoview", True]],
)
def test_e10s_enabling(ConcretePerftest, options, app_name, expected_e10s_flag):
    options["app"] = app_name
    perftest = ConcretePerftest(profile_class="firefox", **options)
    assert perftest.config["e10s"] == expected_e10s_flag


def test_profile_was_provided_locally(ConcretePerftest, options):
    perftest = ConcretePerftest(**options)
    assert os.path.isdir(perftest.config["local_profile_dir"])


@pytest.mark.parametrize(
    "profile_class, app, expected_profile",
    [
        ["firefox", "firefox", "firefox"],
        [None, "firefox", "firefox"],
        ["firefox", None, "firefox"],
    ],
)
def test_profile_class_assignation(
    ConcretePerftest, options, profile_class, app, expected_profile
):
    options["app"] = app
    perftest = ConcretePerftest(profile_class=profile_class, **options)
    assert perftest.profile_class == expected_profile


def test_raptor_venv(ConcretePerftest, options):
    perftest = ConcretePerftest(**options)
    assert perftest.raptor_venv.endswith("raptor-venv")


@pytest.mark.parametrize(
    "run_local,"
    "debug_mode,"
    "post_startup_delay,"
    "expected_post_startup_delay,"
    "expected_debug_mode",
    [
        [True, True, 1234, 1234, True],
        [True, True, 12345, 3000, True],
        [False, False, 1234, 1234, False],
        [False, False, 12345, 12345, False],
        [True, False, 1234, 1234, False],
        [True, False, 12345, 12345, False],
        [False, True, 1234, 1234, False],
        [False, True, 12345, 12345, False],
    ],
)
def test_post_startup_delay(
    ConcretePerftest,
    options,
    run_local,
    debug_mode,
    post_startup_delay,
    expected_post_startup_delay,
    expected_debug_mode,
):
    perftest = ConcretePerftest(
        run_local=run_local,
        debug_mode=debug_mode,
        post_startup_delay=post_startup_delay,
        **options
    )
    assert perftest.post_startup_delay == expected_post_startup_delay
    assert perftest.debug_mode == expected_debug_mode


@pytest.mark.parametrize(
    "alert, expected_alert", [["test_to_alert_on", "test_to_alert_on"], [None, None]]
)
def test_perftest_run_test_setup(
    ConcretePerftest, options, mock_test, alert, expected_alert
):
    perftest = ConcretePerftest(**options)
    mock_test["alert_on"] = alert

    perftest.run_test_setup(mock_test)

    assert perftest.config["subtest_alert_on"] == expected_alert


# WebExtension tests
@pytest.mark.parametrize(
    "app", ["firefox", pytest.mark.xfail("chrome"), pytest.mark.xfail("chromium")]
)
def test_start_browser(get_binary, app):
    binary = get_binary(app)
    assert binary

    raptor = WebExtensionFirefox(app, binary, post_startup_delay=0)

    tests = [{"name": "raptor-{}-tp6".format(app), "page_timeout": 1000}]
    test_names = [test["name"] for test in tests]

    thread = TestBrowserThread(raptor, tests, test_names)
    thread.start()

    timeout = time.time() + 5  # seconds
    while time.time() < timeout:
        try:
            is_running = raptor.runner.is_running()
            assert is_running
            break
        except RunnerNotStartedError:
            time.sleep(0.1)
    else:
        # browser didn't start
        # if the thread had an error, display it here
        thread.print_error()
        assert False

    raptor.clean_up()
    thread.join(5)

    if thread.exc is not None:
        exc, value, tb = thread.exc
        reraise(exc, value, tb)

    assert not raptor.runner.is_running()
    assert raptor.runner.returncode is not None


# Browsertime tests
def test_cmd_arguments(ConcreteBrowsertime, browsertime_options, mock_test):
    expected_cmd = {
        browsertime_options["browsertime_node"],
        browsertime_options["browsertime_browsertimejs"],
        "--firefox.geckodriverPath",
        browsertime_options["browsertime_geckodriver"],
        "--browsertime.page_cycles",
        "1",
        "--browsertime.url",
        mock_test["test_url"],
        "--browsertime.secondary_url",
        mock_test["secondary_url"],
        "--browsertime.page_cycle_delay",
        "1000",
        "--browsertime.post_startup_delay",
        str(DEFAULT_TIMEOUT),
        "--firefox.profileTemplate",
        "--skipHar",
        "--video",
        "true",
        "--visualMetrics",
        "false",
        "--timeouts.pageLoad",
        str(DEFAULT_TIMEOUT),
        "--timeouts.script",
        str(DEFAULT_TIMEOUT),
        "--resultDir",
        "--iterations",
        "1",
    }
    if browsertime_options.get("app") in ["chrome", "chrome-m"]:
        expected_cmd.add(
            "--chrome.chromedriverPath", browsertime_options["browsertime_chromedriver"]
        )

    browsertime = ConcreteBrowsertime(
        post_startup_delay=DEFAULT_TIMEOUT, **browsertime_options
    )
    browsertime.run_test_setup(mock_test)
    cmd = browsertime._compose_cmd(mock_test, DEFAULT_TIMEOUT)

    assert expected_cmd.issubset(set(cmd))


def extract_arg_value(cmd, arg):
    param_index = cmd.index(arg) + 1
    return cmd[param_index]


@pytest.mark.parametrize(
    "arg_to_test, expected, test_patch, options_patch",
    [
        ["--iterations", "1", {}, {"browser_cycles": None}],
        ["--iterations", "123", {"browser_cycles": 123}, {}],
        ["--video", "false", {}, {"browsertime_video": None}],
        ["--video", "true", {}, {"browsertime_video": "dummy_value"}],
        ["--timeouts.script", str(DEFAULT_TIMEOUT), {}, {"page_cycles": None}],
        ["--timeouts.script", str(123 * DEFAULT_TIMEOUT), {"page_cycles": 123}, {}],
        ["--browsertime.page_cycles", "1", {}, {"page_cycles": None}],
        ["--browsertime.page_cycles", "123", {"page_cycles": 123}, {}],
    ],
)
def test_browsertime_arguments(
    ConcreteBrowsertime,
    browsertime_options,
    mock_test,
    arg_to_test,
    expected,
    test_patch,
    options_patch,
):
    mock_test.update(test_patch)
    browsertime_options.update(options_patch)
    browsertime = ConcreteBrowsertime(
        post_startup_delay=DEFAULT_TIMEOUT, **browsertime_options
    )
    browsertime.run_test_setup(mock_test)
    cmd = browsertime._compose_cmd(mock_test, DEFAULT_TIMEOUT)

    param_value = extract_arg_value(cmd, arg_to_test)
    assert expected == param_value


@pytest.mark.parametrize(
    "timeout, expected_timeout, test_patch, options_patch",
    [
        [0, 80, {}, {}],
        [0, 80, {}, {"gecko_profile": False}],
        [1000, 381, {}, {"gecko_profile": True}],
    ],
)
def test_compute_process_timeout(
    ConcreteBrowsertime,
    browsertime_options,
    mock_test,
    timeout,
    expected_timeout,
    test_patch,
    options_patch,
):
    mock_test.update(test_patch)
    browsertime_options.update(options_patch)
    browsertime = ConcreteBrowsertime(
        post_startup_delay=DEFAULT_TIMEOUT, **browsertime_options
    )
    bt_timeout = browsertime._compute_process_timeout(mock_test, timeout, [])
    assert bt_timeout == expected_timeout


@pytest.mark.parametrize(
    "host, playback, benchmark",
    [["127.0.0.1", True, False], ["localhost", False, True]],
)
def test_android_reverse_ports(host, playback, benchmark):
    original_get = WebExtensionAndroid.get_browser_meta
    WebExtensionAndroid.get_browser_meta = mock.MagicMock()
    WebExtensionAndroid.get_browser_meta.return_value = ("app", "version")

    raptor = WebExtensionAndroid(
        "geckoview",
        "org.mozilla.geckoview_example",
        host=host,
        extra_prefs={},
    )
    WebExtensionAndroid.get_browser_meta = original_get

    if benchmark:
        benchmark_mock = mock.patch("raptor.raptor.benchmark.Benchmark")
        raptor.benchmark = benchmark_mock
        raptor.benchmark.port = 1234

    if playback:
        playback_mock = mock.patch(
            "mozbase.mozproxy.mozproxy.backends.mitm.mitm.MitmproxyAndroid"
        )
        playback_mock.port = 4321
        raptor.playback = playback_mock

    raptor.set_reverse_port = Mock()
    raptor.set_reverse_ports()

    raptor.set_reverse_port.assert_any_call(raptor.control_server.port)
    if benchmark:
        raptor.set_reverse_port.assert_any_call(1234)

    if playback:
        raptor.set_reverse_port.assert_any_call(4321)


def test_android_reverse_ports_non_local_host():
    original_get = WebExtensionAndroid.get_browser_meta
    WebExtensionAndroid.get_browser_meta = mock.MagicMock()
    WebExtensionAndroid.get_browser_meta.return_value = ("app", "version")

    raptor = WebExtensionAndroid(
        "geckoview",
        "org.mozilla.geckoview_example",
        host="192.168.100.10",
        extra_prefs={},
    )
    WebExtensionAndroid.get_browser_meta = original_get

    raptor.set_reverse_port = Mock()
    raptor.set_reverse_ports()

    raptor.set_reverse_port.assert_not_called()


if __name__ == "__main__":
    mozunit.main()