diff options
Diffstat (limited to 'layout/tools/reftest/selftest/test_reftest_output.py')
-rw-r--r-- | layout/tools/reftest/selftest/test_reftest_output.py | 162 |
1 files changed, 162 insertions, 0 deletions
diff --git a/layout/tools/reftest/selftest/test_reftest_output.py b/layout/tools/reftest/selftest/test_reftest_output.py new file mode 100644 index 0000000000..ef343f754d --- /dev/null +++ b/layout/tools/reftest/selftest/test_reftest_output.py @@ -0,0 +1,162 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +try: + # Python2 + from cStringIO import StringIO +except ImportError: + # Python3 + from io import StringIO + +from functools import partial + +import mozunit +import pytest +from mozharness.base.log import ERROR, INFO, WARNING +from mozharness.mozilla.automation import TBPL_FAILURE, TBPL_SUCCESS, TBPL_WARNING +from moztest.selftest.output import filter_action, get_mozharness_status + +here = os.path.abspath(os.path.dirname(__file__)) +get_mozharness_status = partial(get_mozharness_status, "reftest") + + +def test_output_pass(runtests): + status, lines = runtests("reftest-pass.list") + assert status == 0 + + tbpl_status, log_level, summary = get_mozharness_status(lines, status) + assert tbpl_status == TBPL_SUCCESS + assert log_level in (INFO, WARNING) + + test_status = filter_action("test_status", lines) + assert len(test_status) == 3 + assert all(t["status"] == "PASS" for t in test_status) + + test_end = filter_action("test_end", lines) + assert len(test_end) == 3 + assert all(t["status"] == "OK" for t in test_end) + + +def test_output_fail(runtests): + formatter = pytest.importorskip("output").ReftestFormatter() + + status, lines = runtests("reftest-fail.list") + assert status == 0 + + buf = StringIO() + tbpl_status, log_level, summary = get_mozharness_status( + lines, status, formatter=formatter, buf=buf + ) + + assert tbpl_status == TBPL_WARNING + assert log_level == WARNING + + test_status = filter_action("test_status", lines) + assert len(test_status) == 3 + assert all(t["status"] == "FAIL" for t in test_status) + assert all("reftest_screenshots" in t["extra"] for t in test_status) + + test_end = filter_action("test_end", lines) + assert len(test_end) == 3 + assert all(t["status"] == "OK" for t in test_end) + + # ensure screenshots were printed + formatted = buf.getvalue() + assert "REFTEST IMAGE 1" in formatted + assert "REFTEST IMAGE 2" in formatted + + +@pytest.mark.skip_mozinfo("!crashreporter") +def test_output_crash(runtests): + status, lines = runtests( + "reftest-crash.list", environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"] + ) + assert status == 245 + + tbpl_status, log_level, summary = get_mozharness_status(lines, status) + assert tbpl_status == TBPL_FAILURE + assert log_level == ERROR + + crash = filter_action("crash", lines) + assert len(crash) == 1 + assert crash[0]["action"] == "crash" + assert crash[0]["signature"] + assert crash[0]["minidump_path"] + + lines = filter_action("test_end", lines) + assert len(lines) == 0 + + +@pytest.mark.skip_mozinfo("!asan") +def test_output_asan(runtests): + status, lines = runtests( + "reftest-crash.list", environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"] + ) + assert status == 245 + + tbpl_status, log_level, summary = get_mozharness_status(lines, status) + assert tbpl_status == TBPL_FAILURE + assert log_level == ERROR + + crash = filter_action("crash", lines) + assert len(crash) == 0 + + process_output = filter_action("process_output", lines) + assert any("ERROR: AddressSanitizer" in l["data"] for l in process_output) + + +@pytest.mark.skip_mozinfo("!debug") +def test_output_assertion(runtests): + status, lines = runtests("reftest-assert.list") + assert status == 0 + + tbpl_status, log_level, summary = get_mozharness_status(lines, status) + assert tbpl_status == TBPL_WARNING + assert log_level == WARNING + + test_status = filter_action("test_status", lines) + assert len(test_status) == 1 + assert test_status[0]["status"] == "PASS" + + test_end = filter_action("test_end", lines) + assert len(test_end) == 1 + assert test_end[0]["status"] == "OK" + + assertions = filter_action("assertion_count", lines) + assert len(assertions) == 1 + assert assertions[0]["count"] == 1 + + +@pytest.mark.skip_mozinfo("!debug") +def test_output_leak(monkeypatch, runtests): + # Monkeypatch mozleak so we always process a failing leak log + # instead of the actual one. + import mozleak + + old_process_leak_log = mozleak.process_leak_log + + def process_leak_log(*args, **kwargs): + return old_process_leak_log( + os.path.join(here, "files", "leaks.log"), *args[1:], **kwargs + ) + + monkeypatch.setattr("mozleak.process_leak_log", process_leak_log) + + status, lines = runtests("reftest-pass.list") + assert status == 0 + + tbpl_status, log_level, summary = get_mozharness_status(lines, status) + assert tbpl_status == TBPL_WARNING + assert log_level == WARNING + + leaks = filter_action("mozleak_total", lines) + assert len(leaks) == 1 + assert leaks[0]["process"] == "default" + assert leaks[0]["bytes"] == 19915 + + +if __name__ == "__main__": + mozunit.main() |