summaryrefslogtreecommitdiffstats
path: root/layout/tools/reftest/selftest
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--layout/tools/reftest/selftest/conftest.py149
-rw-r--r--layout/tools/reftest/selftest/files/assert.html7
-rw-r--r--layout/tools/reftest/selftest/files/crash.html7
-rw-r--r--layout/tools/reftest/selftest/files/defaults.list7
-rw-r--r--layout/tools/reftest/selftest/files/failure-type-interactions.list11
-rw-r--r--layout/tools/reftest/selftest/files/green.html6
-rw-r--r--layout/tools/reftest/selftest/files/invalid-defaults-include.list4
-rw-r--r--layout/tools/reftest/selftest/files/invalid-defaults.list3
-rw-r--r--layout/tools/reftest/selftest/files/invalid-include.list2
-rw-r--r--layout/tools/reftest/selftest/files/leaks.log73
-rw-r--r--layout/tools/reftest/selftest/files/red.html6
-rw-r--r--layout/tools/reftest/selftest/files/reftest-assert.list1
-rw-r--r--layout/tools/reftest/selftest/files/reftest-crash.list1
-rw-r--r--layout/tools/reftest/selftest/files/reftest-fail.list3
-rw-r--r--layout/tools/reftest/selftest/files/reftest-pass.list3
-rw-r--r--layout/tools/reftest/selftest/files/scripttest-pass.html17
-rw-r--r--layout/tools/reftest/selftest/files/types.list5
-rw-r--r--layout/tools/reftest/selftest/python.ini8
-rw-r--r--layout/tools/reftest/selftest/test_python_manifest_parser.py39
-rw-r--r--layout/tools/reftest/selftest/test_reftest_manifest_parser.py74
-rw-r--r--layout/tools/reftest/selftest/test_reftest_output.py164
21 files changed, 590 insertions, 0 deletions
diff --git a/layout/tools/reftest/selftest/conftest.py b/layout/tools/reftest/selftest/conftest.py
new file mode 100644
index 0000000000..74067e9787
--- /dev/null
+++ b/layout/tools/reftest/selftest/conftest.py
@@ -0,0 +1,149 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+import json
+import os
+from argparse import Namespace
+
+try:
+ # Python2
+ from cStringIO import StringIO
+except ImportError:
+ # Python3
+ from io import StringIO
+
+import mozinfo
+import pytest
+from manifestparser import expression
+from moztest.selftest.fixtures import binary, setup_test_harness # noqa
+
+here = os.path.abspath(os.path.dirname(__file__))
+setup_args = [False, "reftest", "reftest"]
+
+
+@pytest.fixture(scope="module")
+def normalize():
+ """A function that can take a relative path and append it to the 'files'
+ directory which contains the data necessary to run these tests.
+ """
+
+ def inner(path):
+ if os.path.isabs(path):
+ return path
+ return os.path.join(here, "files", path)
+
+ return inner
+
+
+@pytest.fixture
+def parser(setup_test_harness):
+ setup_test_harness(*setup_args)
+ cmdline = pytest.importorskip("reftestcommandline")
+ return cmdline.DesktopArgumentsParser()
+
+
+@pytest.fixture
+def get_reftest(setup_test_harness, binary, parser):
+ setup_test_harness(*setup_args)
+ runreftest = pytest.importorskip("runreftest")
+ harness_root = runreftest.SCRIPT_DIRECTORY
+
+ build = parser.build_obj
+ options = vars(parser.parse_args([]))
+ options.update(
+ {
+ "app": binary,
+ "focusFilterMode": "non-needs-focus",
+ "suite": "reftest",
+ }
+ )
+
+ if not os.path.isdir(build.bindir):
+ package_root = os.path.dirname(harness_root)
+ options.update(
+ {
+ "extraProfileFiles": [os.path.join(package_root, "bin", "plugins")],
+ "reftestExtensionPath": os.path.join(harness_root, "reftest"),
+ "sandboxReadWhitelist": [here, os.environ["PYTHON_TEST_TMP"]],
+ "utilityPath": os.path.join(package_root, "bin"),
+ "specialPowersExtensionPath": os.path.join(
+ harness_root, "specialpowers"
+ ),
+ }
+ )
+
+ if "MOZ_FETCHES_DIR" in os.environ:
+ options["sandboxReadWhitelist"].append(os.environ["MOZ_FETCHES_DIR"])
+ else:
+ options.update(
+ {
+ "extraProfileFiles": [os.path.join(build.topobjdir, "dist", "plugins")],
+ "sandboxReadWhitelist": [build.topobjdir, build.topsrcdir],
+ "specialPowersExtensionPath": os.path.join(
+ build.distdir, "xpi-stage", "specialpowers"
+ ),
+ }
+ )
+
+ def inner(**opts):
+ options.update(opts)
+ config = Namespace(**options)
+
+ # This is pulled from `runreftest.run_test_harness` minus some error
+ # checking that isn't necessary in this context. It should stay roughly
+ # in sync.
+ reftest = runreftest.RefTest(config.suite)
+ parser.validate(config, reftest)
+
+ config.app = reftest.getFullPath(config.app)
+ assert os.path.exists(config.app)
+
+ if config.xrePath is None:
+ config.xrePath = os.path.dirname(config.app)
+
+ return reftest, config
+
+ return inner
+
+
+@pytest.fixture # noqa: F811
+def runtests(get_reftest, normalize):
+ def inner(*tests, **opts):
+ assert len(tests) > 0
+ opts["tests"] = map(normalize, tests)
+
+ buf = StringIO()
+ opts["log_raw"] = [buf]
+
+ reftest, options = get_reftest(**opts)
+ result = reftest.runTests(options.tests, options)
+
+ out = json.loads("[" + ",".join(buf.getvalue().splitlines()) + "]")
+ buf.close()
+ return result, out
+
+ return inner
+
+
+@pytest.fixture(autouse=True) # noqa: F811
+def skip_using_mozinfo(request, setup_test_harness):
+ """Gives tests the ability to skip based on values from mozinfo.
+
+ Example:
+ @pytest.mark.skip_mozinfo("!e10s || os == 'linux'")
+ def test_foo():
+ pass
+ """
+
+ setup_test_harness(*setup_args)
+ runreftest = pytest.importorskip("runreftest")
+ runreftest.update_mozinfo()
+
+ skip_mozinfo = request.node.get_marker("skip_mozinfo")
+ if skip_mozinfo:
+ value = skip_mozinfo.args[0]
+ if expression.parse(value, **mozinfo.info):
+ pytest.skip("skipped due to mozinfo match: \n{}".format(value))
diff --git a/layout/tools/reftest/selftest/files/assert.html b/layout/tools/reftest/selftest/files/assert.html
new file mode 100644
index 0000000000..c9aedcf116
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/assert.html
@@ -0,0 +1,7 @@
+<script>
+const Cc = SpecialPowers.Cc;
+const Ci = SpecialPowers.Ci;
+
+let debug = Cc["@mozilla.org/xpcom/debug;1"].getService(Ci.nsIDebug2);
+debug.assertion('failed assertion check', 'false', 'assert.html', 6);
+</script>
diff --git a/layout/tools/reftest/selftest/files/crash.html b/layout/tools/reftest/selftest/files/crash.html
new file mode 100644
index 0000000000..897863024b
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/crash.html
@@ -0,0 +1,7 @@
+<script>
+const Cc = SpecialPowers.Cc;
+const Ci = SpecialPowers.Ci;
+
+let debug = Cc["@mozilla.org/xpcom/debug;1"].getService(Ci.nsIDebug2);
+debug.abort('crash.html', 6);
+</script>
diff --git a/layout/tools/reftest/selftest/files/defaults.list b/layout/tools/reftest/selftest/files/defaults.list
new file mode 100644
index 0000000000..d947eb8d1d
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/defaults.list
@@ -0,0 +1,7 @@
+# test defaults
+defaults pref(foo.bar,true)
+== foo.html foo-ref.html
+
+# reset defaults
+defaults
+== bar.html bar-ref.html
diff --git a/layout/tools/reftest/selftest/files/failure-type-interactions.list b/layout/tools/reftest/selftest/files/failure-type-interactions.list
new file mode 100644
index 0000000000..c820e8f13f
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/failure-type-interactions.list
@@ -0,0 +1,11 @@
+# interactions between skip and fail
+skip-if(true) fails == skip-if_fails.html ref.html
+skip-if(true) fails-if(true) == skip-if_fails-if.html ref.html
+skip fails == skip_fails.html ref.html
+skip-if(false) fails == fails.html ref.html
+fails skip-if(true) == fails_skip-if.html ref.html
+fails-if(true) skip-if(true) == fails-if_skip-if.html ref.html
+fails skip == fails_skip.html ref.html
+fails-if(false) skip == skip.html ref.html
+skip-if(true) fails skip-if(false) == skip-if-true_fails_skip-if-false ref.html
+skip-if(false) fails skip-if(true) == skip-if-false_fails_skip-if-true ref.html
diff --git a/layout/tools/reftest/selftest/files/green.html b/layout/tools/reftest/selftest/files/green.html
new file mode 100644
index 0000000000..d1695cb8b8
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/green.html
@@ -0,0 +1,6 @@
+<!DOCTYPE html>
+<html>
+<body>
+<div style="color: green">Text</div>
+</body>
+</html>
diff --git a/layout/tools/reftest/selftest/files/invalid-defaults-include.list b/layout/tools/reftest/selftest/files/invalid-defaults-include.list
new file mode 100644
index 0000000000..408d3214f6
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/invalid-defaults-include.list
@@ -0,0 +1,4 @@
+# can't use defaults prior to include
+defaults pref(foo.bar,1)
+== foo.html bar.html
+include defaults.list
diff --git a/layout/tools/reftest/selftest/files/invalid-defaults.list b/layout/tools/reftest/selftest/files/invalid-defaults.list
new file mode 100644
index 0000000000..7bb8d060da
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/invalid-defaults.list
@@ -0,0 +1,3 @@
+# invalid tokens in defaults
+defaults skip-if(true) == foo.html bar.html
+== foo.html bar.html
diff --git a/layout/tools/reftest/selftest/files/invalid-include.list b/layout/tools/reftest/selftest/files/invalid-include.list
new file mode 100644
index 0000000000..cd1c1f0939
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/invalid-include.list
@@ -0,0 +1,2 @@
+# non-skip items are not allowed with include
+pref(foo.bar,1) include defaults.list
diff --git a/layout/tools/reftest/selftest/files/leaks.log b/layout/tools/reftest/selftest/files/leaks.log
new file mode 100644
index 0000000000..af832f149d
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/leaks.log
@@ -0,0 +1,73 @@
+== BloatView: ALL (cumulative) LEAK AND BLOAT STATISTICS, default process 1148
+ |<----------------Class--------------->|<-----Bytes------>|<----Objects---->|
+ | | Per-Inst Leaked| Total Rem|
+ 0 |TOTAL | 19 19915|16007885 378|
+ 68 |CacheEntry | 208 208| 521 1|
+ 71 |CacheEntryHandle | 16 16| 7692 1|
+ 72 |CacheFile | 264 264| 521 1|
+ 80 |CacheFileMetadata | 176 176| 521 1|
+ 81 |CacheFileOutputStream | 64 64| 463 1|
+ 90 |CacheStorageService | 256 256| 1 1|
+ 97 |CancelableRunnable | 24 24| 45614 1|
+ 103 |ChannelEventQueue | 96 96| 996 1|
+ 131 |CondVar | 36 360| 822 10|
+ 148 |ConsoleReportCollector | 60 120| 1380 2|
+ 150 |ContentParent | 1712 1712| 2 1|
+ 188 |DataStorage | 284 852| 3 3|
+ 320 |HttpBaseChannel | 1368 1368| 1143 1|
+ 321 |HttpChannelParent | 176 176| 996 1|
+ 322 |HttpChannelParentListener | 48 48| 961 1|
+ 342 |IdlePeriod | 12 36| 166 3|
+ 369 |InterceptedChannelBase | 240 240| 18 1|
+ 389 |LoadContext | 72 144| 1023 2|
+ 391 |LoadInfo | 144 144| 3903 1|
+ 427 |Mutex | 44 1012| 14660 23|
+ 439 |NullPrincipalURI | 80 80| 421 1|
+ 468 |PBrowserParent | 312 312| 21 1|
+ 479 |PContentParent | 1428 1428| 2 1|
+ 486 |PHttpChannelParent | 24 24| 996 1|
+ 527 |PollableEvent | 12 12| 1 1|
+ 576 |ReentrantMonitor | 24 72| 6922 3|
+ 577 |RefCountedMonitor | 84 84| 154 1|
+ 583 |RequestContextService | 60 60| 1 1|
+ 592 |Runnable | 20 40| 178102 2|
+ 627 |Service | 128 128| 1 1|
+ 646 |SharedMemory | 16 16| 1636 1|
+ 675 |StringAdopt | 1 3| 17087 3|
+ 688 |TabParent | 976 976| 21 1|
+ 699 |ThirdPartyUtil | 16 16| 1 1|
+ 875 |ipc::MessageChannel | 208 208| 166 1|
+ 920 |nsAuthURLParser | 12 12| 2 1|
+ 974 |nsCategoryObserver | 72 72| 8 1|
+ 976 |nsChannelClassifier | 28 28| 918 1|
+1004 |CookiePermission | 40 40| 1 1|
+1005 |CookieService | 80 80| 1 1|
+1010 |nsDNSService | 140 140| 1 1|
+1066 |nsEffectiveTLDService | 20 20| 1 1|
+1134 |nsHttpAuthCache::OriginClearObserver | 16 32| 2 2|
+1135 |nsHttpChannel | 1816 1816| 1143 1|
+1136 |nsHttpChannelAuthProvider | 148 148| 1012 1|
+1138 |nsHttpConnectionInfo | 128 128| 1021 1|
+1139 |nsHttpConnectionMgr | 304 304| 1 1|
+1141 |nsHttpHandler | 544 544| 1 1|
+1142 |nsHttpRequestHead | 92 92| 1190 1|
+1145 |nsIDNService | 56 56| 1 1|
+1146 |nsIOService | 176 176| 1 1|
+1176 |nsJSPrincipals | 16 64| 12583 4|
+1186 |nsLocalFile | 88 264| 13423 3|
+1192 |nsMainThreadPtrHolder<T> | 20 80| 2253 4|
+1222 |nsNodeWeakReference | 16 16| 919 1|
+1223 |nsNotifyAddrListener | 112 112| 1 1|
+1241 |PermissionManager | 136 136| 1 1|
+1248 |nsPrefBranch | 76 76| 63 1|
+1257 |nsProxyInfo | 72 72| 1098 1|
+1265 |nsRedirectHistoryEntry | 32 32| 69 1|
+1307 |nsSiteSecurityService | 56 56| 1 1|
+1311 |nsSocketTransportService | 208 208| 1 1|
+1313 |nsStandardURL | 196 1372| 59651 7|
+1319 |nsStreamConverterService | 48 48| 1 1|
+1324 |nsStringBuffer | 8 1688| 722245 211|
+1371 |nsTArray_base | 4 136| 3419841 34|
+1380 |nsThread | 304 912| 165 3|
+1416 |nsWeakReference | 20 180| 1388 9|
+nsTraceRefcnt::DumpStatistics: 1489 entries
diff --git a/layout/tools/reftest/selftest/files/red.html b/layout/tools/reftest/selftest/files/red.html
new file mode 100644
index 0000000000..a9db5be4df
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/red.html
@@ -0,0 +1,6 @@
+<!DOCTYPE html>
+<html>
+<body>
+<div style="color: red">Text</div>
+</body>
+</html>
diff --git a/layout/tools/reftest/selftest/files/reftest-assert.list b/layout/tools/reftest/selftest/files/reftest-assert.list
new file mode 100644
index 0000000000..38fd3f57aa
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/reftest-assert.list
@@ -0,0 +1 @@
+load assert.html
diff --git a/layout/tools/reftest/selftest/files/reftest-crash.list b/layout/tools/reftest/selftest/files/reftest-crash.list
new file mode 100644
index 0000000000..3e27bcba75
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/reftest-crash.list
@@ -0,0 +1 @@
+load crash.html
diff --git a/layout/tools/reftest/selftest/files/reftest-fail.list b/layout/tools/reftest/selftest/files/reftest-fail.list
new file mode 100644
index 0000000000..c5259b6601
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/reftest-fail.list
@@ -0,0 +1,3 @@
+== green.html red.html
+!= green.html green.html
+!= red.html red.html
diff --git a/layout/tools/reftest/selftest/files/reftest-pass.list b/layout/tools/reftest/selftest/files/reftest-pass.list
new file mode 100644
index 0000000000..51512c1202
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/reftest-pass.list
@@ -0,0 +1,3 @@
+== green.html green.html
+== red.html red.html
+!= green.html red.html
diff --git a/layout/tools/reftest/selftest/files/scripttest-pass.html b/layout/tools/reftest/selftest/files/scripttest-pass.html
new file mode 100644
index 0000000000..e0371b3518
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/scripttest-pass.html
@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html>
+<head>
+<title>scripttest-pass</title>
+<script type="text/javascript">
+function getTestCases()
+{
+ return [
+ { testPassed: (function () { return true; }), testDescription: (function () { return "passed"; }) }
+ ];
+}
+</script>
+</head>
+<body>
+<h1>scripttest-pass</h1>
+</body>
+</html>
diff --git a/layout/tools/reftest/selftest/files/types.list b/layout/tools/reftest/selftest/files/types.list
new file mode 100644
index 0000000000..7622564527
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/types.list
@@ -0,0 +1,5 @@
+== green.html green.html
+!= green.html red.html
+load green.html
+script scripttest-pass.html
+print green.html green.html
diff --git a/layout/tools/reftest/selftest/python.ini b/layout/tools/reftest/selftest/python.ini
new file mode 100644
index 0000000000..eb4bbc7e89
--- /dev/null
+++ b/layout/tools/reftest/selftest/python.ini
@@ -0,0 +1,8 @@
+[DEFAULT]
+subsuite=reftest
+sequential=true
+skip-if = python == 3
+
+[test_python_manifest_parser.py]
+[test_reftest_manifest_parser.py]
+[test_reftest_output.py]
diff --git a/layout/tools/reftest/selftest/test_python_manifest_parser.py b/layout/tools/reftest/selftest/test_python_manifest_parser.py
new file mode 100644
index 0000000000..339aadfeb3
--- /dev/null
+++ b/layout/tools/reftest/selftest/test_python_manifest_parser.py
@@ -0,0 +1,39 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+import mozunit
+import pytest
+
+
+@pytest.fixture
+def parse(normalize):
+ reftest = pytest.importorskip("reftest")
+
+ def inner(path):
+ mp = reftest.ReftestManifest()
+ mp.load(normalize(path))
+ return mp
+
+ return inner
+
+
+def test_parse_defaults(parse):
+ mp = parse("defaults.list")
+ assert len(mp.tests) == 4
+
+ for test in mp.tests:
+ if test["name"].startswith("foo"):
+ assert test["pref"] == "foo.bar,true"
+ else:
+ assert "pref" not in test
+
+ # invalid defaults
+ with pytest.raises(ValueError):
+ parse("invalid-defaults.list")
+
+
+if __name__ == "__main__":
+ mozunit.main()
diff --git a/layout/tools/reftest/selftest/test_reftest_manifest_parser.py b/layout/tools/reftest/selftest/test_reftest_manifest_parser.py
new file mode 100644
index 0000000000..c1b715cb28
--- /dev/null
+++ b/layout/tools/reftest/selftest/test_reftest_manifest_parser.py
@@ -0,0 +1,74 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+import mozunit
+import pytest
+
+
+@pytest.fixture
+def parse(get_reftest, normalize):
+ output = pytest.importorskip("output")
+
+ reftest, options = get_reftest(tests=["dummy"])
+ reftest._populate_logger(options)
+ reftest.outputHandler = output.OutputHandler(
+ reftest.log, options.utilityPath, options.symbolsPath
+ )
+
+ def resolve(path):
+ path = normalize(path)
+ return "file://{}".format(path)
+
+ def inner(*manifests):
+ assert len(manifests) > 0
+ manifests = {m: (None, "id") for m in map(resolve, manifests)}
+ return reftest.getActiveTests(manifests, options)
+
+ return inner
+
+
+def test_parse_test_types(parse):
+ tests = parse("types.list")
+ assert tests[0]["type"] == "=="
+ assert tests[1]["type"] == "!="
+ assert tests[2]["type"] == "load"
+ assert tests[3]["type"] == "script"
+ assert tests[4]["type"] == "print"
+
+
+def test_parse_failure_type_interactions(parse):
+ """Tests interactions between skip and fails."""
+ tests = parse("failure-type-interactions.list")
+ for t in tests:
+ if "skip" in t["name"]:
+ assert t["skip"]
+ else:
+ assert not t["skip"]
+
+ # 0 => EXPECTED_PASS, 1 => EXPECTED_FAIL
+ if "fails" in t["name"]:
+ assert t["expected"] == 1
+ else:
+ assert t["expected"] == 0
+
+
+def test_parse_invalid_manifests(parse):
+ # XXX We should assert that the output contains the appropriate error
+ # message, but we seem to be hitting an issue in pytest that is preventing
+ # us from capturing the Gecko output with the capfd fixture. See:
+ # https://github.com/pytest-dev/pytest/issues/5997
+ with pytest.raises(SystemExit):
+ parse("invalid-defaults.list")
+
+ with pytest.raises(SystemExit):
+ parse("invalid-defaults-include.list")
+
+ with pytest.raises(SystemExit):
+ parse("invalid-include.list")
+
+
+if __name__ == "__main__":
+ mozunit.main()
diff --git a/layout/tools/reftest/selftest/test_reftest_output.py b/layout/tools/reftest/selftest/test_reftest_output.py
new file mode 100644
index 0000000000..3f3d8966e0
--- /dev/null
+++ b/layout/tools/reftest/selftest/test_reftest_output.py
@@ -0,0 +1,164 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+import os
+
+try:
+ # Python2
+ from cStringIO import StringIO
+except ImportError:
+ # Python3
+ from io import StringIO
+from functools import partial
+
+import mozunit
+import pytest
+from moztest.selftest.output import get_mozharness_status, filter_action
+
+from mozharness.base.log import INFO, WARNING, ERROR
+from mozharness.mozilla.automation import TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE
+
+here = os.path.abspath(os.path.dirname(__file__))
+get_mozharness_status = partial(get_mozharness_status, "reftest")
+
+
+def test_output_pass(runtests):
+ status, lines = runtests("reftest-pass.list")
+ assert status == 0
+
+ tbpl_status, log_level, summary = get_mozharness_status(lines, status)
+ assert tbpl_status == TBPL_SUCCESS
+ assert log_level in (INFO, WARNING)
+
+ test_status = filter_action("test_status", lines)
+ assert len(test_status) == 3
+ assert all(t["status"] == "PASS" for t in test_status)
+
+ test_end = filter_action("test_end", lines)
+ assert len(test_end) == 3
+ assert all(t["status"] == "OK" for t in test_end)
+
+
+def test_output_fail(runtests):
+ formatter = pytest.importorskip("output").ReftestFormatter()
+
+ status, lines = runtests("reftest-fail.list")
+ assert status == 0
+
+ buf = StringIO()
+ tbpl_status, log_level, summary = get_mozharness_status(
+ lines, status, formatter=formatter, buf=buf
+ )
+
+ assert tbpl_status == TBPL_WARNING
+ assert log_level == WARNING
+
+ test_status = filter_action("test_status", lines)
+ assert len(test_status) == 3
+ assert all(t["status"] == "FAIL" for t in test_status)
+ assert all("reftest_screenshots" in t["extra"] for t in test_status)
+
+ test_end = filter_action("test_end", lines)
+ assert len(test_end) == 3
+ assert all(t["status"] == "OK" for t in test_end)
+
+ # ensure screenshots were printed
+ formatted = buf.getvalue()
+ assert "REFTEST IMAGE 1" in formatted
+ assert "REFTEST IMAGE 2" in formatted
+
+
+@pytest.mark.skip_mozinfo("!crashreporter")
+def test_output_crash(runtests):
+ status, lines = runtests(
+ "reftest-crash.list", environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"]
+ )
+ assert status == 1
+
+ tbpl_status, log_level, summary = get_mozharness_status(lines, status)
+ assert tbpl_status == TBPL_FAILURE
+ assert log_level == ERROR
+
+ crash = filter_action("crash", lines)
+ assert len(crash) == 1
+ assert crash[0]["action"] == "crash"
+ assert crash[0]["signature"]
+ assert crash[0]["minidump_path"]
+
+ lines = filter_action("test_end", lines)
+ assert len(lines) == 0
+
+
+@pytest.mark.skip_mozinfo("!asan")
+def test_output_asan(runtests):
+ status, lines = runtests(
+ "reftest-crash.list", environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"]
+ )
+ assert status == 0
+
+ tbpl_status, log_level, summary = get_mozharness_status(lines, status)
+ assert tbpl_status == TBPL_FAILURE
+ assert log_level == ERROR
+
+ crash = filter_action("crash", lines)
+ assert len(crash) == 0
+
+ process_output = filter_action("process_output", lines)
+ assert any("ERROR: AddressSanitizer" in l["data"] for l in process_output)
+
+
+@pytest.mark.skip_mozinfo("!debug")
+def test_output_assertion(runtests):
+ status, lines = runtests("reftest-assert.list")
+ assert status == 0
+
+ tbpl_status, log_level, summary = get_mozharness_status(lines, status)
+ assert tbpl_status == TBPL_WARNING
+ assert log_level == WARNING
+
+ test_status = filter_action("test_status", lines)
+ assert len(test_status) == 1
+ assert test_status[0]["status"] == "PASS"
+
+ test_end = filter_action("test_end", lines)
+ assert len(test_end) == 1
+ assert test_end[0]["status"] == "OK"
+
+ assertions = filter_action("assertion_count", lines)
+ assert len(assertions) == 1
+ assert assertions[0]["count"] == 1
+
+
+@pytest.mark.skip_mozinfo("!debug")
+def test_output_leak(monkeypatch, runtests):
+ # Monkeypatch mozleak so we always process a failing leak log
+ # instead of the actual one.
+ import mozleak
+
+ old_process_leak_log = mozleak.process_leak_log
+
+ def process_leak_log(*args, **kwargs):
+ return old_process_leak_log(
+ os.path.join(here, "files", "leaks.log"), *args[1:], **kwargs
+ )
+
+ monkeypatch.setattr("mozleak.process_leak_log", process_leak_log)
+
+ status, lines = runtests("reftest-pass.list")
+ assert status == 0
+
+ tbpl_status, log_level, summary = get_mozharness_status(lines, status)
+ assert tbpl_status == TBPL_WARNING
+ assert log_level == WARNING
+
+ leaks = filter_action("mozleak_total", lines)
+ assert len(leaks) == 1
+ assert leaks[0]["process"] == "default"
+ assert leaks[0]["bytes"] == 19915
+
+
+if __name__ == "__main__":
+ mozunit.main()