summaryrefslogtreecommitdiffstats
path: root/testing/web-platform/tests/tools/runner/report.py
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 09:22:09 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 09:22:09 +0000
commit43a97878ce14b72f0981164f87f2e35e14151312 (patch)
tree620249daf56c0258faa40cbdcf9cfba06de2a846 /testing/web-platform/tests/tools/runner/report.py
parentInitial commit. (diff)
downloadfirefox-upstream.tar.xz
firefox-upstream.zip
Adding upstream version 110.0.1.upstream/110.0.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'testing/web-platform/tests/tools/runner/report.py')
-rw-r--r--testing/web-platform/tests/tools/runner/report.py308
1 files changed, 308 insertions, 0 deletions
diff --git a/testing/web-platform/tests/tools/runner/report.py b/testing/web-platform/tests/tools/runner/report.py
new file mode 100644
index 0000000000..3dc6cfb8bb
--- /dev/null
+++ b/testing/web-platform/tests/tools/runner/report.py
@@ -0,0 +1,308 @@
+# flake8: noqa
+# mypy: ignore-errors
+
+import argparse
+import json
+import sys
+import types
+
+from cgi import escape
+from collections import defaultdict
+
+
+def html_escape(item, escape_quote=False):
+ if isinstance(item, types.StringTypes):
+ rv = escape(item)
+ if escape_quote:
+ rv = rv.replace('"', "&quot;")
+ return rv
+ else:
+ return item
+
+
+class Raw:
+ """Simple wrapper around a string to stop it being escaped by html_escape"""
+ def __init__(self, value):
+ self.value = value
+
+ def __unicode__(self):
+ return unicode(self.value)
+
+
+class Node:
+ """Node structure used when building HTML"""
+ def __init__(self, name, attrs, children):
+ #Need list of void elements
+ self.name = name
+ self.attrs = attrs
+ self.children = children
+
+ def __unicode__(self):
+ if self.attrs:
+ #Need to escape
+ attrs_unicode = " " + " ".join("%s=\"%s\"" % (html_escape(key),
+ html_escape(value,
+ escape_quote=True))
+ for key, value in self.attrs.items())
+ else:
+ attrs_unicode = ""
+ return "<%s%s>%s</%s>\n" % (self.name,
+ attrs_unicode,
+ "".join(unicode(html_escape(item))
+ for item in self.children),
+ self.name)
+
+ def __str__(self):
+ return unicode(self).encode("utf8")
+
+
+class RootNode:
+ """Special Node representing the document root"""
+ def __init__(self, *children):
+ self.children = ["<!DOCTYPE html>"] + list(children)
+
+ def __unicode__(self):
+ return "".join(unicode(item) for item in self.children)
+
+ def __str__(self):
+ return unicode(self).encode("utf8")
+
+
+def flatten(iterable):
+ """Flatten a list of lists by one level so that
+ [1,["abc"], "def",[2, [3]]]
+ becomes
+ [1, "abc", "def", 2, [3]]"""
+ rv = []
+ for item in iterable:
+ if hasattr(item, "__iter__") and not isinstance(item, types.StringTypes):
+ rv.extend(item)
+ else:
+ rv.append(item)
+ return rv
+
+
+class HTML:
+ """Simple HTML templating system. An instance of this class can create
+ element nodes by calling methods with the same name as the element,
+ passing in children as positional arguments or as a list, and attributes
+ as keyword arguments, with _ replacing - and trailing _ for python keywords
+
+ e.g.
+
+ h = HTML()
+ print(h.html(
+ html.head(),
+ html.body([html.h1("Hello World!")], class_="body-class")
+ ))
+ Would give
+ <!DOCTYPE html><html><head></head><body class="body-class"><h1>Hello World!</h1></body></html>"""
+ def __getattr__(self, name):
+ def make_html(self, *content, **attrs):
+ for attr_name in attrs.keys():
+ if "_" in attr_name:
+ new_name = attr_name.replace("_", "-")
+ if new_name.endswith("-"):
+ new_name = new_name[:-1]
+ attrs[new_name] = attrs.pop(attr_name)
+ return Node(name, attrs, flatten(content))
+
+ method = types.MethodType(make_html, self, HTML)
+ setattr(self, name, method)
+ return method
+
+ def __call__(self, *children):
+ return RootNode(*flatten(children))
+
+
+h = HTML()
+
+
+class TestResult:
+ """Simple holder for the results of a single test in a single UA"""
+ def __init__(self, test):
+ self.test = test
+ self.results = {}
+
+ def __cmp__(self, other):
+ return self.test == other.test
+
+ def __hash__(self):
+ return hash(self.test)
+
+
+def load_data(args):
+ """Load data treating args as a list of UA name, filename pairs"""
+ pairs = []
+ for i in xrange(0, len(args), 2):
+ pairs.append(args[i:i+2])
+
+ rv = {}
+ for UA, filename in pairs:
+ with open(filename) as f:
+ rv[UA] = json.load(f)
+
+ return rv
+
+
+def test_id(id):
+ """Convert a test id in JSON into an immutable object that
+ can be used as a dictionary key"""
+ if isinstance(id, list):
+ return tuple(id)
+ else:
+ return id
+
+
+def all_tests(data):
+ tests = defaultdict(set)
+ for UA, results in iteritems(data):
+ for result in results["results"]:
+ id = test_id(result["test"])
+ tests[id] |= {subtest["name"] for subtest in result["subtests"]}
+ return tests
+
+
+def group_results(data):
+ """Produce a list of UAs and a dictionary mapping specific tests to their
+ status in all UAs e.g.
+ ["UA1", "UA2"], {"test_id":{"harness":{"UA1": (status1, message1),
+ "UA2": (status2, message2)},
+ "subtests":{"subtest1": "UA1": (status1-1, message1-1),
+ "UA2": (status2-1, message2-1)}}}
+ Status and message are None if the test didn't run in a particular UA.
+ Message is None if the test didn't produce a message"""
+ tests = all_tests(data)
+
+ UAs = data.keys()
+
+ def result():
+ return {
+ "harness": {UA: (None, None) for UA in UAs},
+ "subtests": None # init this later
+ }
+
+ results_by_test = defaultdict(result)
+
+ for UA, results in iteritems(data):
+ for test_data in results["results"]:
+ id = test_id(test_data["test"])
+ result = results_by_test[id]
+
+ if result["subtests"] is None:
+ result["subtests"] = {
+ name: {UA: (None, None) for UA in UAs} for name in tests[id]
+ }
+
+ result["harness"][UA] = (test_data["status"], test_data["message"])
+ for subtest in test_data["subtests"]:
+ result["subtests"][subtest["name"]][UA] = (subtest["status"],
+ subtest["message"])
+
+ return UAs, results_by_test
+
+
+def status_cell(status, message=None):
+ """Produce a table cell showing the status of a test"""
+ status = status if status is not None else "NONE"
+ kwargs = {}
+ if message:
+ kwargs["title"] = message
+ status_text = status.title()
+ return h.td(status_text, class_="status " + status,
+ **kwargs)
+
+
+def test_link(test_id, subtest=None):
+ """Produce an <a> element linking to a test"""
+ if isinstance(test_id, types.StringTypes):
+ rv = [h.a(test_id, href=test_id)]
+ else:
+ rv = [h.a(test_id[0], href=test_id[0]),
+ " %s " % test_id[1],
+ h.a(test_id[2], href=test_id[2])]
+ if subtest is not None:
+ rv.append(" [%s]" % subtest)
+ return rv
+
+
+def summary(UAs, results_by_test):
+ """Render the implementation report summary"""
+ not_passing = []
+ for test, results in iteritems(results_by_test):
+ if not any(item[0] in ("PASS", "OK") for item in results["harness"].values()):
+ not_passing.append((test, None))
+ for subtest_name, subtest_results in iteritems(results["subtests"]):
+ if not any(item[0] == "PASS" for item in subtest_results.values()):
+ not_passing.append((test, subtest_name))
+ if not_passing:
+ rv = [
+ h.p("The following tests failed to pass in all UAs:"),
+ h.ul([h.li(test_link(test, subtest))
+ for test, subtest in not_passing])
+ ]
+ else:
+ rv = "All tests passed in at least one UA"
+ return rv
+
+
+def result_rows(UAs, test, result):
+ """Render the results for each test run"""
+ yield h.tr(
+ h.td(
+ test_link(test),
+ rowspan=(1 + len(result["subtests"]))
+ ),
+ h.td(),
+ [status_cell(status, message)
+ for UA, (status, message) in sorted(result["harness"].items())],
+ class_="test"
+ )
+
+ for name, subtest_result in sorted(iteritems(result["subtests"])):
+ yield h.tr(
+ h.td(name),
+ [status_cell(status, message)
+ for UA, (status, message) in sorted(subtest_result.items())],
+ class_="subtest"
+ )
+
+
+def result_bodies(UAs, results_by_test):
+ return [h.tbody(result_rows(UAs, test, result))
+ for test, result in sorted(iteritems(results_by_test))]
+
+
+def generate_html(UAs, results_by_test):
+ """Generate all the HTML output"""
+ return h(h.html(
+ h.head(
+ h.meta(charset="utf8"),
+ h.title("Implementation Report"),
+ h.link(href="report.css", rel="stylesheet")),
+ h.body(
+ h.h1("Implementation Report"),
+ h.h2("Summary"),
+ summary(UAs, results_by_test),
+ h.h2("Full Results"),
+ h.table(
+ h.thead(
+ h.tr(
+ h.th("Test"),
+ h.th("Subtest"),
+ [h.th(UA) for UA in sorted(UAs)])),
+ result_bodies(UAs, results_by_test)))))
+
+
+def main(filenames):
+ data = load_data(filenames)
+ UAs, results_by_test = group_results(data)
+ return generate_html(UAs, results_by_test)
+
+
+if __name__ == "__main__":
+ if not sys.argv[1:]:
+ print("""Please supply a list of UA name, filename pairs e.g.
+
+python report.py Firefox firefox.json Chrome chrome.json IE internet_explorer.json""")
+ print(main(sys.argv[1:]))
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-30 00:44:28 +0000