1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
# -*- coding: utf-8 -*-
import json
import time
import mozunit
import pytest
# flake8: noqa
@pytest.mark.parametrize(
"logs,expected",
(
pytest.param(
[
(
"suite_start",
{
"manifestA": ["test_foo", "test_bar", "test_baz"],
"manifestB": ["test_something"],
},
),
("test_start", "test_foo"),
("test_end", "test_foo", "SKIP"),
("test_start", "test_bar"),
("test_end", "test_bar", "OK"),
("test_start", "test_something"),
("test_end", "test_something", "OK"),
("test_start", "test_baz"),
("test_end", "test_baz", "PASS", "FAIL"),
("suite_end",),
],
"""
{"groups": ["manifestA", "manifestB"], "action": "test_groups", "line": 0}
{"test": "test_baz", "subtest": null, "group": "manifestA", "status": "PASS", "expected": "FAIL", "message": null, "stack": null, "known_intermittent": [], "action": "test_result", "line": 8}
{"group": "manifestA", "status": "ERROR", "duration": 70, "action": "group_result", "line": 9}
{"group": "manifestB", "status": "OK", "duration": 10, "action": "group_result", "line": 9}
""".strip(),
id="basic",
),
pytest.param(
[
("suite_start", {"manifest": ["test_foo"]}),
("test_start", "test_foo"),
("suite_end",),
],
"""
{"groups": ["manifest"], "action": "test_groups", "line": 0}
{"group": "manifest", "status": null, "duration": null, "action": "group_result", "line": 2}
""".strip(),
id="missing_test_end",
),
pytest.param(
[
("suite_start", {"manifest": ["test_foo"]}),
("test_start", "test_foo"),
("test_status", "test_foo", "subtest", "PASS"),
("suite_end",),
],
"""
{"groups": ["manifest"], "action": "test_groups", "line": 0}
{"group": "manifest", "status": "ERROR", "duration": null, "action": "group_result", "line": 3}
""".strip(),
id="missing_test_end_with_test_status_ok",
marks=pytest.mark.xfail, # status is OK but should be ERROR
),
pytest.param(
[
(
"suite_start",
{
"manifestA": ["test_foo", "test_bar", "test_baz"],
"manifestB": ["test_something"],
},
),
("test_start", "test_foo"),
("test_end", "test_foo", "SKIP"),
("test_start", "test_bar"),
("test_end", "test_bar", "CRASH"),
("test_start", "test_something"),
("test_end", "test_something", "OK"),
("test_start", "test_baz"),
("test_end", "test_baz", "FAIL", "FAIL"),
("suite_end",),
],
"""
{"groups": ["manifestA", "manifestB"], "action": "test_groups", "line": 0}
{"test": "test_bar", "subtest": null, "group": "manifestA", "status": "CRASH", "expected": "OK", "message": null, "stack": null, "known_intermittent": [], "action": "test_result", "line": 4}
{"group": "manifestA", "status": "ERROR", "duration": 70, "action": "group_result", "line": 9}
{"group": "manifestB", "status": "OK", "duration": 10, "action": "group_result", "line": 9}
""".strip(),
id="crash_and_group_status",
),
),
)
def test_errorsummary(monkeypatch, get_logger, logs, expected):
ts = {"ts": 0.0} # need to use dict since 'nonlocal' doesn't exist on PY2
def fake_time():
ts["ts"] += 0.01
return ts["ts"]
monkeypatch.setattr(time, "time", fake_time)
logger = get_logger("errorsummary")
for log in logs:
getattr(logger, log[0])(*log[1:])
buf = logger.handlers[0].stream
result = buf.getvalue()
print("Dumping result for copy/paste:")
print(result)
expected = expected.split("\n")
for i, line in enumerate(result.split("\n")):
if not line:
continue
data = json.loads(line)
assert data == json.loads(expected[i])
if __name__ == "__main__":
mozunit.main()
|
