From 0ebf5bdf043a27fd3dfb7f92e0cb63d88954c44d Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 03:47:29 +0200 Subject: Adding upstream version 115.8.0esr. Signed-off-by: Daniel Baumann --- layout/tools/reftest/selftest/conftest.py | 147 +++++++++++++++++++ layout/tools/reftest/selftest/files/assert.html | 7 + layout/tools/reftest/selftest/files/crash.html | 7 + layout/tools/reftest/selftest/files/defaults.list | 7 + .../selftest/files/failure-type-interactions.list | 11 ++ layout/tools/reftest/selftest/files/green.html | 6 + .../selftest/files/invalid-defaults-include.list | 4 + .../reftest/selftest/files/invalid-defaults.list | 3 + .../reftest/selftest/files/invalid-include.list | 2 + layout/tools/reftest/selftest/files/leaks.log | 73 ++++++++++ layout/tools/reftest/selftest/files/red.html | 6 + .../reftest/selftest/files/reftest-assert.list | 1 + .../reftest/selftest/files/reftest-crash.list | 1 + .../tools/reftest/selftest/files/reftest-fail.list | 3 + .../tools/reftest/selftest/files/reftest-pass.list | 3 + .../reftest/selftest/files/scripttest-pass.html | 17 +++ layout/tools/reftest/selftest/files/types.list | 5 + layout/tools/reftest/selftest/python.ini | 7 + .../selftest/test_python_manifest_parser.py | 37 +++++ .../selftest/test_reftest_manifest_parser.py | 72 +++++++++ .../tools/reftest/selftest/test_reftest_output.py | 162 +++++++++++++++++++++ 21 files changed, 581 insertions(+) create mode 100644 layout/tools/reftest/selftest/conftest.py create mode 100644 layout/tools/reftest/selftest/files/assert.html create mode 100644 layout/tools/reftest/selftest/files/crash.html create mode 100644 layout/tools/reftest/selftest/files/defaults.list create mode 100644 layout/tools/reftest/selftest/files/failure-type-interactions.list create mode 100644 layout/tools/reftest/selftest/files/green.html create mode 100644 layout/tools/reftest/selftest/files/invalid-defaults-include.list create mode 100644 layout/tools/reftest/selftest/files/invalid-defaults.list create mode 100644 layout/tools/reftest/selftest/files/invalid-include.list create mode 100644 layout/tools/reftest/selftest/files/leaks.log create mode 100644 layout/tools/reftest/selftest/files/red.html create mode 100644 layout/tools/reftest/selftest/files/reftest-assert.list create mode 100644 layout/tools/reftest/selftest/files/reftest-crash.list create mode 100644 layout/tools/reftest/selftest/files/reftest-fail.list create mode 100644 layout/tools/reftest/selftest/files/reftest-pass.list create mode 100644 layout/tools/reftest/selftest/files/scripttest-pass.html create mode 100644 layout/tools/reftest/selftest/files/types.list create mode 100644 layout/tools/reftest/selftest/python.ini create mode 100644 layout/tools/reftest/selftest/test_python_manifest_parser.py create mode 100644 layout/tools/reftest/selftest/test_reftest_manifest_parser.py create mode 100644 layout/tools/reftest/selftest/test_reftest_output.py (limited to 'layout/tools/reftest/selftest') diff --git a/layout/tools/reftest/selftest/conftest.py b/layout/tools/reftest/selftest/conftest.py new file mode 100644 index 0000000000..1255caf8a7 --- /dev/null +++ b/layout/tools/reftest/selftest/conftest.py @@ -0,0 +1,147 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +from argparse import Namespace + +try: + # Python2 + from cStringIO import StringIO +except ImportError: + # Python3 + from io import StringIO + +import mozinfo +import pytest +from manifestparser import expression +from moztest.selftest.fixtures import binary_fixture, setup_test_harness # noqa + +here = os.path.abspath(os.path.dirname(__file__)) +setup_args = [False, "reftest", "reftest"] + + +@pytest.fixture(scope="module") +def normalize(): + """A function that can take a relative path and append it to the 'files' + directory which contains the data necessary to run these tests. + """ + + def inner(path): + if os.path.isabs(path): + return path + return os.path.join(here, "files", path) + + return inner + + +@pytest.fixture +def parser(setup_test_harness): + setup_test_harness(*setup_args) + cmdline = pytest.importorskip("reftestcommandline") + return cmdline.DesktopArgumentsParser() + + +@pytest.fixture +def get_reftest(setup_test_harness, binary, parser): + setup_test_harness(*setup_args) + runreftest = pytest.importorskip("runreftest") + harness_root = runreftest.SCRIPT_DIRECTORY + + build = parser.build_obj + options = vars(parser.parse_args([])) + options.update( + { + "app": binary, + "focusFilterMode": "non-needs-focus", + "suite": "reftest", + } + ) + + if not os.path.isdir(build.bindir): + package_root = os.path.dirname(harness_root) + options.update( + { + "extraProfileFiles": [os.path.join(package_root, "bin", "plugins")], + "reftestExtensionPath": os.path.join(harness_root, "reftest"), + "sandboxReadWhitelist": [here, os.environ["PYTHON_TEST_TMP"]], + "utilityPath": os.path.join(package_root, "bin"), + "specialPowersExtensionPath": os.path.join( + harness_root, "specialpowers" + ), + } + ) + + if "MOZ_FETCHES_DIR" in os.environ: + options["sandboxReadWhitelist"].append(os.environ["MOZ_FETCHES_DIR"]) + else: + options.update( + { + "extraProfileFiles": [os.path.join(build.topobjdir, "dist", "plugins")], + "sandboxReadWhitelist": [build.topobjdir, build.topsrcdir], + "specialPowersExtensionPath": os.path.join( + build.distdir, "xpi-stage", "specialpowers" + ), + } + ) + + def inner(**opts): + options.update(opts) + config = Namespace(**options) + + # This is pulled from `runreftest.run_test_harness` minus some error + # checking that isn't necessary in this context. It should stay roughly + # in sync. + reftest = runreftest.RefTest(config.suite) + parser.validate(config, reftest) + + config.app = reftest.getFullPath(config.app) + assert os.path.exists(config.app) + + if config.xrePath is None: + config.xrePath = os.path.dirname(config.app) + + return reftest, config + + return inner + + +@pytest.fixture # noqa: F811 +def runtests(get_reftest, normalize): + def inner(*tests, **opts): + assert len(tests) > 0 + opts["tests"] = map(normalize, tests) + + buf = StringIO() + opts["log_raw"] = [buf] + + reftest, options = get_reftest(**opts) + result = reftest.runTests(options.tests, options) + + out = json.loads("[" + ",".join(buf.getvalue().splitlines()) + "]") + buf.close() + return result, out + + return inner + + +@pytest.fixture(autouse=True) # noqa: F811 +def skip_using_mozinfo(request, setup_test_harness): + """Gives tests the ability to skip based on values from mozinfo. + + Example: + @pytest.mark.skip_mozinfo("!e10s || os == 'linux'") + def test_foo(): + pass + """ + + setup_test_harness(*setup_args) + runreftest = pytest.importorskip("runreftest") + runreftest.update_mozinfo() + + skip_mozinfo = request.node.get_closest_marker("skip_mozinfo") + if skip_mozinfo: + value = skip_mozinfo.args[0] + if expression.parse(value, **mozinfo.info): + pytest.skip("skipped due to mozinfo match: \n{}".format(value)) diff --git a/layout/tools/reftest/selftest/files/assert.html b/layout/tools/reftest/selftest/files/assert.html new file mode 100644 index 0000000000..c9aedcf116 --- /dev/null +++ b/layout/tools/reftest/selftest/files/assert.html @@ -0,0 +1,7 @@ + diff --git a/layout/tools/reftest/selftest/files/crash.html b/layout/tools/reftest/selftest/files/crash.html new file mode 100644 index 0000000000..897863024b --- /dev/null +++ b/layout/tools/reftest/selftest/files/crash.html @@ -0,0 +1,7 @@ + diff --git a/layout/tools/reftest/selftest/files/defaults.list b/layout/tools/reftest/selftest/files/defaults.list new file mode 100644 index 0000000000..d947eb8d1d --- /dev/null +++ b/layout/tools/reftest/selftest/files/defaults.list @@ -0,0 +1,7 @@ +# test defaults +defaults pref(foo.bar,true) +== foo.html foo-ref.html + +# reset defaults +defaults +== bar.html bar-ref.html diff --git a/layout/tools/reftest/selftest/files/failure-type-interactions.list b/layout/tools/reftest/selftest/files/failure-type-interactions.list new file mode 100644 index 0000000000..c820e8f13f --- /dev/null +++ b/layout/tools/reftest/selftest/files/failure-type-interactions.list @@ -0,0 +1,11 @@ +# interactions between skip and fail +skip-if(true) fails == skip-if_fails.html ref.html +skip-if(true) fails-if(true) == skip-if_fails-if.html ref.html +skip fails == skip_fails.html ref.html +skip-if(false) fails == fails.html ref.html +fails skip-if(true) == fails_skip-if.html ref.html +fails-if(true) skip-if(true) == fails-if_skip-if.html ref.html +fails skip == fails_skip.html ref.html +fails-if(false) skip == skip.html ref.html +skip-if(true) fails skip-if(false) == skip-if-true_fails_skip-if-false ref.html +skip-if(false) fails skip-if(true) == skip-if-false_fails_skip-if-true ref.html diff --git a/layout/tools/reftest/selftest/files/green.html b/layout/tools/reftest/selftest/files/green.html new file mode 100644 index 0000000000..d1695cb8b8 --- /dev/null +++ b/layout/tools/reftest/selftest/files/green.html @@ -0,0 +1,6 @@ + + + +
Text
+ + diff --git a/layout/tools/reftest/selftest/files/invalid-defaults-include.list b/layout/tools/reftest/selftest/files/invalid-defaults-include.list new file mode 100644 index 0000000000..408d3214f6 --- /dev/null +++ b/layout/tools/reftest/selftest/files/invalid-defaults-include.list @@ -0,0 +1,4 @@ +# can't use defaults prior to include +defaults pref(foo.bar,1) +== foo.html bar.html +include defaults.list diff --git a/layout/tools/reftest/selftest/files/invalid-defaults.list b/layout/tools/reftest/selftest/files/invalid-defaults.list new file mode 100644 index 0000000000..7bb8d060da --- /dev/null +++ b/layout/tools/reftest/selftest/files/invalid-defaults.list @@ -0,0 +1,3 @@ +# invalid tokens in defaults +defaults skip-if(true) == foo.html bar.html +== foo.html bar.html diff --git a/layout/tools/reftest/selftest/files/invalid-include.list b/layout/tools/reftest/selftest/files/invalid-include.list new file mode 100644 index 0000000000..cd1c1f0939 --- /dev/null +++ b/layout/tools/reftest/selftest/files/invalid-include.list @@ -0,0 +1,2 @@ +# non-skip items are not allowed with include +pref(foo.bar,1) include defaults.list diff --git a/layout/tools/reftest/selftest/files/leaks.log b/layout/tools/reftest/selftest/files/leaks.log new file mode 100644 index 0000000000..af832f149d --- /dev/null +++ b/layout/tools/reftest/selftest/files/leaks.log @@ -0,0 +1,73 @@ +== BloatView: ALL (cumulative) LEAK AND BLOAT STATISTICS, default process 1148 + |<----------------Class--------------->|<-----Bytes------>|<----Objects---->| + | | Per-Inst Leaked| Total Rem| + 0 |TOTAL | 19 19915|16007885 378| + 68 |CacheEntry | 208 208| 521 1| + 71 |CacheEntryHandle | 16 16| 7692 1| + 72 |CacheFile | 264 264| 521 1| + 80 |CacheFileMetadata | 176 176| 521 1| + 81 |CacheFileOutputStream | 64 64| 463 1| + 90 |CacheStorageService | 256 256| 1 1| + 97 |CancelableRunnable | 24 24| 45614 1| + 103 |ChannelEventQueue | 96 96| 996 1| + 131 |CondVar | 36 360| 822 10| + 148 |ConsoleReportCollector | 60 120| 1380 2| + 150 |ContentParent | 1712 1712| 2 1| + 188 |DataStorage | 284 852| 3 3| + 320 |HttpBaseChannel | 1368 1368| 1143 1| + 321 |HttpChannelParent | 176 176| 996 1| + 322 |HttpChannelParentListener | 48 48| 961 1| + 342 |IdlePeriod | 12 36| 166 3| + 369 |InterceptedChannelBase | 240 240| 18 1| + 389 |LoadContext | 72 144| 1023 2| + 391 |LoadInfo | 144 144| 3903 1| + 427 |Mutex | 44 1012| 14660 23| + 439 |NullPrincipalURI | 80 80| 421 1| + 468 |PBrowserParent | 312 312| 21 1| + 479 |PContentParent | 1428 1428| 2 1| + 486 |PHttpChannelParent | 24 24| 996 1| + 527 |PollableEvent | 12 12| 1 1| + 576 |ReentrantMonitor | 24 72| 6922 3| + 577 |RefCountedMonitor | 84 84| 154 1| + 583 |RequestContextService | 60 60| 1 1| + 592 |Runnable | 20 40| 178102 2| + 627 |Service | 128 128| 1 1| + 646 |SharedMemory | 16 16| 1636 1| + 675 |StringAdopt | 1 3| 17087 3| + 688 |TabParent | 976 976| 21 1| + 699 |ThirdPartyUtil | 16 16| 1 1| + 875 |ipc::MessageChannel | 208 208| 166 1| + 920 |nsAuthURLParser | 12 12| 2 1| + 974 |nsCategoryObserver | 72 72| 8 1| + 976 |nsChannelClassifier | 28 28| 918 1| +1004 |CookiePermission | 40 40| 1 1| +1005 |CookieService | 80 80| 1 1| +1010 |nsDNSService | 140 140| 1 1| +1066 |nsEffectiveTLDService | 20 20| 1 1| +1134 |nsHttpAuthCache::OriginClearObserver | 16 32| 2 2| +1135 |nsHttpChannel | 1816 1816| 1143 1| +1136 |nsHttpChannelAuthProvider | 148 148| 1012 1| +1138 |nsHttpConnectionInfo | 128 128| 1021 1| +1139 |nsHttpConnectionMgr | 304 304| 1 1| +1141 |nsHttpHandler | 544 544| 1 1| +1142 |nsHttpRequestHead | 92 92| 1190 1| +1145 |nsIDNService | 56 56| 1 1| +1146 |nsIOService | 176 176| 1 1| +1176 |nsJSPrincipals | 16 64| 12583 4| +1186 |nsLocalFile | 88 264| 13423 3| +1192 |nsMainThreadPtrHolder | 20 80| 2253 4| +1222 |nsNodeWeakReference | 16 16| 919 1| +1223 |nsNotifyAddrListener | 112 112| 1 1| +1241 |PermissionManager | 136 136| 1 1| +1248 |nsPrefBranch | 76 76| 63 1| +1257 |nsProxyInfo | 72 72| 1098 1| +1265 |nsRedirectHistoryEntry | 32 32| 69 1| +1307 |nsSiteSecurityService | 56 56| 1 1| +1311 |nsSocketTransportService | 208 208| 1 1| +1313 |nsStandardURL | 196 1372| 59651 7| +1319 |nsStreamConverterService | 48 48| 1 1| +1324 |nsStringBuffer | 8 1688| 722245 211| +1371 |nsTArray_base | 4 136| 3419841 34| +1380 |nsThread | 304 912| 165 3| +1416 |nsWeakReference | 20 180| 1388 9| +nsTraceRefcnt::DumpStatistics: 1489 entries diff --git a/layout/tools/reftest/selftest/files/red.html b/layout/tools/reftest/selftest/files/red.html new file mode 100644 index 0000000000..a9db5be4df --- /dev/null +++ b/layout/tools/reftest/selftest/files/red.html @@ -0,0 +1,6 @@ + + + +
Text
+ + diff --git a/layout/tools/reftest/selftest/files/reftest-assert.list b/layout/tools/reftest/selftest/files/reftest-assert.list new file mode 100644 index 0000000000..38fd3f57aa --- /dev/null +++ b/layout/tools/reftest/selftest/files/reftest-assert.list @@ -0,0 +1 @@ +load assert.html diff --git a/layout/tools/reftest/selftest/files/reftest-crash.list b/layout/tools/reftest/selftest/files/reftest-crash.list new file mode 100644 index 0000000000..3e27bcba75 --- /dev/null +++ b/layout/tools/reftest/selftest/files/reftest-crash.list @@ -0,0 +1 @@ +load crash.html diff --git a/layout/tools/reftest/selftest/files/reftest-fail.list b/layout/tools/reftest/selftest/files/reftest-fail.list new file mode 100644 index 0000000000..c5259b6601 --- /dev/null +++ b/layout/tools/reftest/selftest/files/reftest-fail.list @@ -0,0 +1,3 @@ +== green.html red.html +!= green.html green.html +!= red.html red.html diff --git a/layout/tools/reftest/selftest/files/reftest-pass.list b/layout/tools/reftest/selftest/files/reftest-pass.list new file mode 100644 index 0000000000..51512c1202 --- /dev/null +++ b/layout/tools/reftest/selftest/files/reftest-pass.list @@ -0,0 +1,3 @@ +== green.html green.html +== red.html red.html +!= green.html red.html diff --git a/layout/tools/reftest/selftest/files/scripttest-pass.html b/layout/tools/reftest/selftest/files/scripttest-pass.html new file mode 100644 index 0000000000..e0371b3518 --- /dev/null +++ b/layout/tools/reftest/selftest/files/scripttest-pass.html @@ -0,0 +1,17 @@ + + + +scripttest-pass + + + +

scripttest-pass

+ + diff --git a/layout/tools/reftest/selftest/files/types.list b/layout/tools/reftest/selftest/files/types.list new file mode 100644 index 0000000000..7622564527 --- /dev/null +++ b/layout/tools/reftest/selftest/files/types.list @@ -0,0 +1,5 @@ +== green.html green.html +!= green.html red.html +load green.html +script scripttest-pass.html +print green.html green.html diff --git a/layout/tools/reftest/selftest/python.ini b/layout/tools/reftest/selftest/python.ini new file mode 100644 index 0000000000..6c4dd9c713 --- /dev/null +++ b/layout/tools/reftest/selftest/python.ini @@ -0,0 +1,7 @@ +[DEFAULT] +subsuite=reftest +sequential=true + +[test_python_manifest_parser.py] +[test_reftest_manifest_parser.py] +[test_reftest_output.py] diff --git a/layout/tools/reftest/selftest/test_python_manifest_parser.py b/layout/tools/reftest/selftest/test_python_manifest_parser.py new file mode 100644 index 0000000000..5ec9ce324c --- /dev/null +++ b/layout/tools/reftest/selftest/test_python_manifest_parser.py @@ -0,0 +1,37 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozunit +import pytest + + +@pytest.fixture +def parse(normalize): + reftest = pytest.importorskip("reftest") + + def inner(path): + mp = reftest.ReftestManifest() + mp.load(normalize(path)) + return mp + + return inner + + +def test_parse_defaults(parse): + mp = parse("defaults.list") + assert len(mp.tests) == 4 + + for test in mp.tests: + if test["name"].startswith("foo"): + assert test["pref"] == "foo.bar,true" + else: + assert "pref" not in test + + # invalid defaults + with pytest.raises(ValueError): + parse("invalid-defaults.list") + + +if __name__ == "__main__": + mozunit.main() diff --git a/layout/tools/reftest/selftest/test_reftest_manifest_parser.py b/layout/tools/reftest/selftest/test_reftest_manifest_parser.py new file mode 100644 index 0000000000..009aa17a7f --- /dev/null +++ b/layout/tools/reftest/selftest/test_reftest_manifest_parser.py @@ -0,0 +1,72 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozunit +import pytest + + +@pytest.fixture +def parse(get_reftest, normalize): + output = pytest.importorskip("output") + + reftest, options = get_reftest(tests=["dummy"]) + reftest._populate_logger(options) + reftest.outputHandler = output.OutputHandler( + reftest.log, options.utilityPath, options.symbolsPath + ) + + def resolve(path): + path = normalize(path) + return "file://{}".format(path) + + def inner(*manifests): + assert len(manifests) > 0 + manifests = {m: (None, "id") for m in map(resolve, manifests)} + return reftest.getActiveTests(manifests, options) + + return inner + + +def test_parse_test_types(parse): + tests = parse("types.list") + assert tests[0]["type"] == "==" + assert tests[1]["type"] == "!=" + assert tests[2]["type"] == "load" + assert tests[3]["type"] == "script" + assert tests[4]["type"] == "print" + + +def test_parse_failure_type_interactions(parse): + """Tests interactions between skip and fails.""" + tests = parse("failure-type-interactions.list") + for t in tests: + if "skip" in t["name"]: + assert t["skip"] + else: + assert not t["skip"] + + # 0 => EXPECTED_PASS, 1 => EXPECTED_FAIL + if "fails" in t["name"]: + assert t["expected"] == 1 + else: + assert t["expected"] == 0 + + +def test_parse_invalid_manifests(parse): + # XXX We should assert that the output contains the appropriate error + # message, but we seem to be hitting an issue in pytest that is preventing + # us from capturing the Gecko output with the capfd fixture. See: + # https://github.com/pytest-dev/pytest/issues/5997 + with pytest.raises(SystemExit): + parse("invalid-defaults.list") + + with pytest.raises(SystemExit): + parse("invalid-defaults-include.list") + + with pytest.raises(SystemExit): + parse("invalid-include.list") + + +if __name__ == "__main__": + mozunit.main() diff --git a/layout/tools/reftest/selftest/test_reftest_output.py b/layout/tools/reftest/selftest/test_reftest_output.py new file mode 100644 index 0000000000..ef343f754d --- /dev/null +++ b/layout/tools/reftest/selftest/test_reftest_output.py @@ -0,0 +1,162 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +try: + # Python2 + from cStringIO import StringIO +except ImportError: + # Python3 + from io import StringIO + +from functools import partial + +import mozunit +import pytest +from mozharness.base.log import ERROR, INFO, WARNING +from mozharness.mozilla.automation import TBPL_FAILURE, TBPL_SUCCESS, TBPL_WARNING +from moztest.selftest.output import filter_action, get_mozharness_status + +here = os.path.abspath(os.path.dirname(__file__)) +get_mozharness_status = partial(get_mozharness_status, "reftest") + + +def test_output_pass(runtests): + status, lines = runtests("reftest-pass.list") + assert status == 0 + + tbpl_status, log_level, summary = get_mozharness_status(lines, status) + assert tbpl_status == TBPL_SUCCESS + assert log_level in (INFO, WARNING) + + test_status = filter_action("test_status", lines) + assert len(test_status) == 3 + assert all(t["status"] == "PASS" for t in test_status) + + test_end = filter_action("test_end", lines) + assert len(test_end) == 3 + assert all(t["status"] == "OK" for t in test_end) + + +def test_output_fail(runtests): + formatter = pytest.importorskip("output").ReftestFormatter() + + status, lines = runtests("reftest-fail.list") + assert status == 0 + + buf = StringIO() + tbpl_status, log_level, summary = get_mozharness_status( + lines, status, formatter=formatter, buf=buf + ) + + assert tbpl_status == TBPL_WARNING + assert log_level == WARNING + + test_status = filter_action("test_status", lines) + assert len(test_status) == 3 + assert all(t["status"] == "FAIL" for t in test_status) + assert all("reftest_screenshots" in t["extra"] for t in test_status) + + test_end = filter_action("test_end", lines) + assert len(test_end) == 3 + assert all(t["status"] == "OK" for t in test_end) + + # ensure screenshots were printed + formatted = buf.getvalue() + assert "REFTEST IMAGE 1" in formatted + assert "REFTEST IMAGE 2" in formatted + + +@pytest.mark.skip_mozinfo("!crashreporter") +def test_output_crash(runtests): + status, lines = runtests( + "reftest-crash.list", environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"] + ) + assert status == 245 + + tbpl_status, log_level, summary = get_mozharness_status(lines, status) + assert tbpl_status == TBPL_FAILURE + assert log_level == ERROR + + crash = filter_action("crash", lines) + assert len(crash) == 1 + assert crash[0]["action"] == "crash" + assert crash[0]["signature"] + assert crash[0]["minidump_path"] + + lines = filter_action("test_end", lines) + assert len(lines) == 0 + + +@pytest.mark.skip_mozinfo("!asan") +def test_output_asan(runtests): + status, lines = runtests( + "reftest-crash.list", environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"] + ) + assert status == 245 + + tbpl_status, log_level, summary = get_mozharness_status(lines, status) + assert tbpl_status == TBPL_FAILURE + assert log_level == ERROR + + crash = filter_action("crash", lines) + assert len(crash) == 0 + + process_output = filter_action("process_output", lines) + assert any("ERROR: AddressSanitizer" in l["data"] for l in process_output) + + +@pytest.mark.skip_mozinfo("!debug") +def test_output_assertion(runtests): + status, lines = runtests("reftest-assert.list") + assert status == 0 + + tbpl_status, log_level, summary = get_mozharness_status(lines, status) + assert tbpl_status == TBPL_WARNING + assert log_level == WARNING + + test_status = filter_action("test_status", lines) + assert len(test_status) == 1 + assert test_status[0]["status"] == "PASS" + + test_end = filter_action("test_end", lines) + assert len(test_end) == 1 + assert test_end[0]["status"] == "OK" + + assertions = filter_action("assertion_count", lines) + assert len(assertions) == 1 + assert assertions[0]["count"] == 1 + + +@pytest.mark.skip_mozinfo("!debug") +def test_output_leak(monkeypatch, runtests): + # Monkeypatch mozleak so we always process a failing leak log + # instead of the actual one. + import mozleak + + old_process_leak_log = mozleak.process_leak_log + + def process_leak_log(*args, **kwargs): + return old_process_leak_log( + os.path.join(here, "files", "leaks.log"), *args[1:], **kwargs + ) + + monkeypatch.setattr("mozleak.process_leak_log", process_leak_log) + + status, lines = runtests("reftest-pass.list") + assert status == 0 + + tbpl_status, log_level, summary = get_mozharness_status(lines, status) + assert tbpl_status == TBPL_WARNING + assert log_level == WARNING + + leaks = filter_action("mozleak_total", lines) + assert len(leaks) == 1 + assert leaks[0]["process"] == "default" + assert leaks[0]["bytes"] == 19915 + + +if __name__ == "__main__": + mozunit.main() -- cgit v1.2.3