1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
|
#!/usr/bin/python3
from __future__ import print_function
import argparse
import json
import sys
from pathlib import Path
import yaml
def parse_args(cmdln_args):
parser = argparse.ArgumentParser(description="Parse UI test logs an results")
parser.add_argument(
"--output-md",
type=argparse.FileType("w", encoding="utf-8"),
help="Output markdown file.",
required=True,
)
parser.add_argument(
"--log",
type=argparse.FileType("r", encoding="utf-8"),
help="Log output of flank.",
required=True,
)
parser.add_argument(
"--results", type=Path, help="Directory containing flank results", required=True
)
parser.add_argument(
"--exit-code", type=int, help="Exit code of flank.", required=True
)
parser.add_argument("--device-type", help="Type of device ", required=True)
parser.add_argument(
"--report-treeherder-failures",
help="Report failures in treeherder format.",
required=False,
action="store_true",
)
return parser.parse_args(args=cmdln_args)
def extract_android_args(log):
return yaml.safe_load(log.split("AndroidArgs\n")[1].split("RunTests\n")[0])
def main():
args = parse_args(sys.argv[1:])
log = args.log.read()
matrix_ids = json.loads(args.results.joinpath("matrix_ids.json").read_text())
android_args = extract_android_args(log)
print = args.output_md.write
print("# Devices\n")
print(yaml.safe_dump(android_args["gcloud"]["device"]))
print("# Results\n")
print("| Matrix | Result | Firebase Test Lab | Details\n")
print("| --- | --- | --- | --- |\n")
for matrix, matrix_result in matrix_ids.items():
for axis in matrix_result["axes"]:
print(
f"| {matrix_result['matrixId']} | {matrix_result['outcome']}"
f"| [Firebase Test Lab]({matrix_result['webLink']}) | {axis['details']}\n"
)
if (
args.report_treeherder_failures
and matrix_result["outcome"] != "success"
and matrix_result["outcome"] != "flaky"
):
# write failures to test log in format known to treeherder logviewer
sys.stdout.write(
f"TEST-UNEXPECTED-FAIL | {matrix_result['outcome']} | {matrix_result['webLink']} | {axis['details']}\n"
)
print("---\n")
print("# References & Documentation\n")
print(
"* [Automated UI Testing Documentation](https://github.com/mozilla-mobile/shared-docs/blob/main/android/ui-testing.md)\n"
)
print(
"* Mobile Test Engineering on [Confluence](https://mozilla-hub.atlassian.net/wiki/spaces/MTE/overview) | [Slack](https://mozilla.slack.com/archives/C02KDDS9QM9) | [Alerts](https://mozilla.slack.com/archives/C0134KJ4JHL)\n"
)
if __name__ == "__main__":
main()
|
