diff options
Diffstat (limited to 'testing/testinfo.py')
| -rw-r--r-- | testing/testinfo.py | 956 |
1 files changed, 956 insertions, 0 deletions
diff --git a/testing/testinfo.py b/testing/testinfo.py new file mode 100644 index 0000000000..e17926f85e --- /dev/null +++ b/testing/testinfo.py @@ -0,0 +1,956 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import datetime +import errno +import json +import os +import posixpath +import re +import subprocess +from collections import defaultdict + +import mozpack.path as mozpath +import requests +import six.moves.urllib_parse as urlparse +from mozbuild.base import MachCommandConditions as conditions +from mozbuild.base import MozbuildObject +from mozfile import which +from moztest.resolve import TestManifestLoader, TestResolver +from redo import retriable + +REFERER = "https://wiki.developer.mozilla.org/en-US/docs/Mozilla/Test-Info" +MAX_DAYS = 30 + + +class TestInfo(object): + """ + Support 'mach test-info'. + """ + + def __init__(self, verbose): + self.verbose = verbose + here = os.path.abspath(os.path.dirname(__file__)) + self.build_obj = MozbuildObject.from_environment(cwd=here) + + def log_verbose(self, what): + if self.verbose: + print(what) + + +class TestInfoTests(TestInfo): + """ + Support 'mach test-info tests': Detailed report of specified tests. + """ + + def __init__(self, verbose): + TestInfo.__init__(self, verbose) + + self._hg = None + if conditions.is_hg(self.build_obj): + self._hg = which("hg") + if not self._hg: + raise OSError(errno.ENOENT, "Could not find 'hg' on PATH.") + + self._git = None + if conditions.is_git(self.build_obj): + self._git = which("git") + if not self._git: + raise OSError(errno.ENOENT, "Could not find 'git' on PATH.") + + def find_in_hg_or_git(self, test_name): + if self._hg: + cmd = [self._hg, "files", "-I", test_name] + elif self._git: + cmd = [self._git, "ls-files", test_name] + else: + return None + try: + out = subprocess.check_output(cmd, universal_newlines=True).splitlines() + except subprocess.CalledProcessError: + out = None + return out + + def set_test_name(self): + # Generating a unified report for a specific test is complicated + # by differences in the test name used in various data sources. + # Consider: + # - It is often convenient to request a report based only on + # a short file name, rather than the full path; + # - Bugs may be filed in bugzilla against a simple, short test + # name or the full path to the test; + # This function attempts to find appropriate names for different + # queries based on the specified test name. + + # full_test_name is full path to file in hg (or git) + self.full_test_name = None + out = self.find_in_hg_or_git(self.test_name) + if out and len(out) == 1: + self.full_test_name = out[0] + elif out and len(out) > 1: + print("Ambiguous test name specified. Found:") + for line in out: + print(line) + else: + out = self.find_in_hg_or_git("**/%s*" % self.test_name) + if out and len(out) == 1: + self.full_test_name = out[0] + elif out and len(out) > 1: + print("Ambiguous test name. Found:") + for line in out: + print(line) + if self.full_test_name: + self.full_test_name.replace(os.sep, posixpath.sep) + print("Found %s in source control." % self.full_test_name) + else: + print("Unable to validate test name '%s'!" % self.test_name) + self.full_test_name = self.test_name + + # search for full_test_name in test manifests + here = os.path.abspath(os.path.dirname(__file__)) + resolver = TestResolver.from_environment( + cwd=here, loader_cls=TestManifestLoader + ) + relpath = self.build_obj._wrap_path_argument(self.full_test_name).relpath() + tests = list(resolver.resolve_tests(paths=[relpath])) + if len(tests) == 1: + relpath = self.build_obj._wrap_path_argument(tests[0]["manifest"]).relpath() + print("%s found in manifest %s" % (self.full_test_name, relpath)) + if tests[0].get("flavor"): + print(" flavor: %s" % tests[0]["flavor"]) + if tests[0].get("skip-if"): + print(" skip-if: %s" % tests[0]["skip-if"]) + if tests[0].get("fail-if"): + print(" fail-if: %s" % tests[0]["fail-if"]) + elif len(tests) == 0: + print("%s not found in any test manifest!" % self.full_test_name) + else: + print("%s found in more than one manifest!" % self.full_test_name) + + # short_name is full_test_name without path + self.short_name = None + name_idx = self.full_test_name.rfind("/") + if name_idx > 0: + self.short_name = self.full_test_name[name_idx + 1 :] + if self.short_name and self.short_name == self.test_name: + self.short_name = None + + def get_platform(self, record): + if "platform" in record["build"]: + platform = record["build"]["platform"] + else: + platform = "-" + platform_words = platform.split("-") + types_label = "" + # combine run and build types and eliminate duplicates + run_types = [] + if "run" in record and "type" in record["run"]: + run_types = record["run"]["type"] + run_types = run_types if isinstance(run_types, list) else [run_types] + build_types = [] + if "build" in record and "type" in record["build"]: + build_types = record["build"]["type"] + build_types = ( + build_types if isinstance(build_types, list) else [build_types] + ) + run_types = list(set(run_types + build_types)) + # '1proc' is used as a treeherder label but does not appear in run types + if "e10s" not in run_types: + run_types = run_types + ["1proc"] + for run_type in run_types: + # chunked is not interesting + if run_type == "chunked": + continue + # e10s is the default: implied + if run_type == "e10s": + continue + # sometimes a build/run type is already present in the build platform + if run_type in platform_words: + continue + if types_label: + types_label += "-" + types_label += run_type + return "%s/%s:" % (platform, types_label) + + def report_bugs(self): + # Report open bugs matching test name + search = self.full_test_name + if self.test_name: + search = "%s,%s" % (search, self.test_name) + if self.short_name: + search = "%s,%s" % (search, self.short_name) + payload = {"quicksearch": search, "include_fields": "id,summary"} + response = requests.get("https://bugzilla.mozilla.org/rest/bug", payload) + response.raise_for_status() + json_response = response.json() + print("\nBugzilla quick search for '%s':" % search) + if "bugs" in json_response: + for bug in json_response["bugs"]: + print("Bug %s: %s" % (bug["id"], bug["summary"])) + else: + print("No bugs found.") + + def report( + self, + test_names, + start, + end, + show_info, + show_bugs, + ): + self.start = start + self.end = end + self.show_info = show_info + + if not self.show_info and not show_bugs: + # by default, show everything + self.show_info = True + show_bugs = True + + for test_name in test_names: + print("===== %s =====" % test_name) + self.test_name = test_name + if len(self.test_name) < 6: + print("'%s' is too short for a test name!" % self.test_name) + continue + self.set_test_name() + if show_bugs: + self.report_bugs() + + +class TestInfoReport(TestInfo): + """ + Support 'mach test-info report': Report of test runs summarized by + manifest and component. + """ + + def __init__(self, verbose): + TestInfo.__init__(self, verbose) + self.threads = [] + + @retriable(attempts=3, sleeptime=5, sleepscale=2) + def get_url(self, target_url): + # if we fail to get valid json (i.e. end point has malformed data), return {} + retVal = {} + try: + self.log_verbose("getting url: %s" % target_url) + r = requests.get(target_url, headers={"User-agent": "mach-test-info/1.0"}) + self.log_verbose("got status: %s" % r.status_code) + r.raise_for_status() + retVal = r.json() + except json.decoder.JSONDecodeError: + self.log_verbose("Error retrieving data from %s" % target_url) + + return retVal + + def update_report(self, by_component, result, path_mod): + def update_item(item, label, value): + # It is important to include any existing item value in case ActiveData + # returns multiple records for the same test; that can happen if the report + # sometimes maps more than one ActiveData record to the same path. + new_value = item.get(label, 0) + value + if type(new_value) == int: + item[label] = new_value + else: + item[label] = float(round(new_value, 2)) # pylint: disable=W1633 + + if "test" in result and "tests" in by_component: + test = result["test"] + if path_mod: + test = path_mod(test) + for bc in by_component["tests"]: + for item in by_component["tests"][bc]: + if test == item["test"]: + # pylint: disable=W1633 + seconds = float(round(result.get("duration", 0), 2)) + update_item(item, "total run time, seconds", seconds) + update_item(item, "total runs", result.get("count", 0)) + update_item(item, "skipped runs", result.get("skips", 0)) + update_item(item, "failed runs", result.get("failures", 0)) + return True + return False + + def path_mod_reftest(self, path): + # "<path1> == <path2>" -> "<path1>" + path = path.split(" ")[0] + # "<path>?<params>" -> "<path>" + path = path.split("?")[0] + # "<path>#<fragment>" -> "<path>" + path = path.split("#")[0] + return path + + def path_mod_jsreftest(self, path): + # "<path>;assert" -> "<path>" + path = path.split(";")[0] + return path + + def path_mod_marionette(self, path): + # "<path> <test-name>" -> "<path>" + path = path.split(" ")[0] + # "part1\part2" -> "part1/part2" + path = path.replace("\\", os.path.sep) + return path + + def path_mod_wpt(self, path): + if path[0] == os.path.sep: + # "/<path>" -> "<path>" + path = path[1:] + # "<path>" -> "testing/web-platform/tests/<path>" + path = os.path.join("testing", "web-platform", "tests", path) + # "<path>?<params>" -> "<path>" + path = path.split("?")[0] + return path + + def path_mod_jittest(self, path): + # "part1\part2" -> "part1/part2" + path = path.replace("\\", os.path.sep) + # "<path>" -> "js/src/jit-test/tests/<path>" + return os.path.join("js", "src", "jit-test", "tests", path) + + def path_mod_xpcshell(self, path): + # <manifest>.{ini|toml}:<path> -> "<path>" + path = path.split(":")[-1] + return path + + def description( + self, + components, + flavor, + subsuite, + paths, + show_manifests, + show_tests, + show_summary, + show_annotations, + filter_values, + filter_keys, + start_date, + end_date, + ): + # provide a natural language description of the report options + what = [] + if show_manifests: + what.append("test manifests") + if show_tests: + what.append("tests") + if show_annotations: + what.append("test manifest annotations") + if show_summary and len(what) == 0: + what.append("summary of tests only") + if len(what) > 1: + what[-1] = "and " + what[-1] + what = ", ".join(what) + d = "Test summary report for " + what + if components: + d += ", in specified components (%s)" % components + else: + d += ", in all components" + if flavor: + d += ", in specified flavor (%s)" % flavor + if subsuite: + d += ", in specified subsuite (%s)" % subsuite + if paths: + d += ", on specified paths (%s)" % paths + if filter_values: + d += ", containing '%s'" % filter_values + if filter_keys: + d += " in manifest keys '%s'" % filter_keys + else: + d += " in any part of manifest entry" + d += ", including historical run-time data for the last " + + start = datetime.datetime.strptime(start_date, "%Y-%m-%d") + end = datetime.datetime.strptime(end_date, "%Y-%m-%d") + d += "%s days on trunk (autoland/m-c)" % ((end - start).days) + d += " as of %s." % end_date + return d + + # TODO: this is hacked for now and very limited + def parse_test(self, summary): + if summary.endswith("single tracking bug"): + name_part = summary.split("|")[0] # remove 'single tracking bug' + name_part.strip() + return name_part.split()[-1] # get just the test name, not extra words + return None + + def get_runcount_data(self, runcounts_input_file, start, end): + # TODO: use start/end properly + if runcounts_input_file: + try: + with open(runcounts_input_file, "r") as f: + runcounts = json.load(f) + except: + print("Unable to load runcounts from path: %s" % runcounts_input_file) + raise + else: + runcounts = self.get_runcounts(days=MAX_DAYS) + runcounts = self.squash_runcounts(runcounts, days=MAX_DAYS) + return runcounts + + def get_testinfoall_index_url(self): + import taskcluster + + index = taskcluster.Index( + { + "rootUrl": "https://firefox-ci-tc.services.mozilla.com", + } + ) + route = "gecko.v2.mozilla-central.latest.source.test-info-all" + queue = taskcluster.Queue( + { + "rootUrl": "https://firefox-ci-tc.services.mozilla.com", + } + ) + + task_id = index.findTask(route)["taskId"] + artifacts = queue.listLatestArtifacts(task_id)["artifacts"] + + url = "" + for artifact in artifacts: + if artifact["name"].endswith("test-run-info.json"): + url = queue.buildUrl("getLatestArtifact", task_id, artifact["name"]) + break + return url + + def get_runcounts(self, days=MAX_DAYS): + testrundata = {} + # get historical data from test-info job artifact; if missing get fresh + url = self.get_testinfoall_index_url() + print("INFO: requesting runcounts url: %s" % url) + olddata = self.get_url(url) + + # fill in any holes we have + endday = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta( + days=1 + ) + startday = endday - datetime.timedelta(days=days) + urls_to_fetch = [] + # build list of dates with missing data + while startday < endday: + nextday = startday + datetime.timedelta(days=1) + if not olddata.get(str(nextday.date()), {}): + url = "https://treeherder.mozilla.org/api/groupsummary/" + url += "?startdate=%s&enddate=%s" % ( + startday.date(), + nextday.date(), + ) + urls_to_fetch.append([str(nextday.date()), url]) + testrundata[str(nextday.date())] = olddata.get(str(nextday.date()), {}) + + startday = nextday + + # limit missing data collection to 5 most recent days days to reduce overall runtime + for date, url in urls_to_fetch[-5:]: + try: + testrundata[date] = self.get_url(url) + except requests.exceptions.HTTPError: + # We want to see other errors, but can accept HTTPError failures + print(f"Unable to retrieve results for url: {url}") + pass + + return testrundata + + def squash_runcounts(self, runcounts, days=MAX_DAYS): + # squash all testrundata together into 1 big happy family for the last X days + endday = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta( + days=1 + ) + oldest = endday - datetime.timedelta(days=days) + + testgroup_runinfo = defaultdict(lambda: defaultdict(int)) + + retVal = {} + for datekey in runcounts.keys(): + # strip out older days + if datetime.date.fromisoformat(datekey) < oldest.date(): + continue + + jtn = runcounts[datekey].get("job_type_names", {}) + if not jtn: + print("Warning: Missing job type names from date: %s" % datekey) + continue + + for m in runcounts[datekey]["manifests"]: + man_name = list(m.keys())[0] + + for job_type_id, result, classification, count in m[man_name]: + # format: job_type_name, result, classification, count + # find matching jtn, result, classification and increment 'count' + job_name = jtn[job_type_id] + key = (job_name, result, classification) + testgroup_runinfo[man_name][key] += count + + for m in testgroup_runinfo: + retVal[m] = [ + list(x) + [testgroup_runinfo[m][x]] for x in testgroup_runinfo[m] + ] + return retVal + + def get_intermittent_failure_data(self, start, end): + retVal = {} + + # get IFV bug list + # i.e. https://th.m.o/api/failures/?startday=2022-06-22&endday=2022-06-29&tree=all + url = ( + "https://treeherder.mozilla.org/api/failures/?startday=%s&endday=%s&tree=trunk" + % (start, end) + ) + if_data = self.get_url(url) + buglist = [x["bug_id"] for x in if_data] + + # get bug data for summary, 800 bugs at a time + # i.e. https://b.m.o/rest/bug?include_fields=id,product,component,summary&id=1,2,3... + max_bugs = 800 + bug_data = [] + fields = ["id", "product", "component", "summary"] + for bug_index in range(0, len(buglist), max_bugs): + bugs = [str(x) for x in buglist[bug_index : bug_index + max_bugs]] + if not bugs: + print(f"warning: found no bugs in range {bug_index}, +{max_bugs}") + continue + + url = "https://bugzilla.mozilla.org/rest/bug?include_fields=%s&id=%s" % ( + ",".join(fields), + ",".join(bugs), + ) + data = self.get_url(url) + if data and "bugs" in data.keys(): + bug_data.extend(data["bugs"]) + + # for each summary, parse filename, store component + # IF we find >1 bug with same testname, for now summarize as one + for bug in bug_data: + test_name = self.parse_test(bug["summary"]) + if not test_name: + continue + + c = int([x["bug_count"] for x in if_data if x["bug_id"] == bug["id"]][0]) + if test_name not in retVal.keys(): + retVal[test_name] = { + "id": bug["id"], + "count": 0, + "product": bug["product"], + "component": bug["component"], + } + retVal[test_name]["count"] += c + + if bug["product"] != retVal[test_name]["product"]: + print( + "ERROR | %s | mismatched bugzilla product, bugzilla (%s) != repo (%s)" + % (bug["id"], bug["product"], retVal[test_name]["product"]) + ) + if bug["component"] != retVal[test_name]["component"]: + print( + "ERROR | %s | mismatched bugzilla component, bugzilla (%s) != repo (%s)" + % (bug["id"], bug["component"], retVal[test_name]["component"]) + ) + return retVal + + def report( + self, + components, + flavor, + subsuite, + paths, + show_manifests, + show_tests, + show_summary, + show_annotations, + filter_values, + filter_keys, + show_components, + output_file, + start, + end, + show_testruns, + runcounts_input_file, + ): + def matches_filters(test): + """ + Return True if all of the requested filter_values are found in this test; + if filter_keys are specified, restrict search to those test keys. + """ + for value in filter_values: + value_found = False + for key in test: + if not filter_keys or key in filter_keys: + if re.search(value, test[key]): + value_found = True + break + if not value_found: + return False + return True + + start_time = datetime.datetime.now() + + # Ensure useful report by default + if ( + not show_manifests + and not show_tests + and not show_summary + and not show_annotations + ): + show_manifests = True + show_summary = True + + by_component = {} + if components: + components = components.split(",") + if filter_keys: + filter_keys = filter_keys.split(",") + if filter_values: + filter_values = filter_values.split(",") + else: + filter_values = [] + display_keys = (filter_keys or []) + ["skip-if", "fail-if", "fails-if"] + display_keys = set(display_keys) + ifd = self.get_intermittent_failure_data(start, end) + + runcount = {} + if show_testruns and os.environ.get("GECKO_HEAD_REPOSITORY", "") in [ + "https://hg.mozilla.org/mozilla-central", + "https://hg.mozilla.org/try", + ]: + runcount = self.get_runcount_data(runcounts_input_file, start, end) + + print("Finding tests...") + here = os.path.abspath(os.path.dirname(__file__)) + resolver = TestResolver.from_environment( + cwd=here, loader_cls=TestManifestLoader + ) + tests = list( + resolver.resolve_tests(paths=paths, flavor=flavor, subsuite=subsuite) + ) + + manifest_paths = set() + for t in tests: + if t.get("manifest", None): + manifest_path = t["manifest"] + if t.get("ancestor_manifest", None): + manifest_path = "%s:%s" % (t["ancestor_manifest"], t["manifest"]) + manifest_paths.add(manifest_path) + manifest_count = len(manifest_paths) + print( + "Resolver found {} tests, {} manifests".format(len(tests), manifest_count) + ) + + if show_manifests: + topsrcdir = self.build_obj.topsrcdir + by_component["manifests"] = {} + manifest_paths = list(manifest_paths) + manifest_paths.sort() + relpaths = [] + for manifest_path in manifest_paths: + relpath = mozpath.relpath(manifest_path, topsrcdir) + if mozpath.commonprefix((manifest_path, topsrcdir)) != topsrcdir: + continue + relpaths.append(relpath) + reader = self.build_obj.mozbuild_reader(config_mode="empty") + files_info = reader.files_info(relpaths) + for manifest_path in manifest_paths: + relpath = mozpath.relpath(manifest_path, topsrcdir) + if mozpath.commonprefix((manifest_path, topsrcdir)) != topsrcdir: + continue + manifest_info = None + if relpath in files_info: + bug_component = files_info[relpath].get("BUG_COMPONENT") + if bug_component: + key = "{}::{}".format( + bug_component.product, bug_component.component + ) + else: + key = "<unknown bug component>" + if (not components) or (key in components): + manifest_info = {"manifest": relpath, "tests": 0, "skipped": 0} + rkey = key if show_components else "all" + if rkey in by_component["manifests"]: + by_component["manifests"][rkey].append(manifest_info) + else: + by_component["manifests"][rkey] = [manifest_info] + if manifest_info: + for t in tests: + if t["manifest"] == manifest_path: + manifest_info["tests"] += 1 + if t.get("skip-if"): + manifest_info["skipped"] += 1 + for key in by_component["manifests"]: + by_component["manifests"][key].sort(key=lambda k: k["manifest"]) + + if show_tests: + by_component["tests"] = {} + + if show_tests or show_summary or show_annotations: + test_count = 0 + failed_count = 0 + skipped_count = 0 + annotation_count = 0 + condition_count = 0 + component_set = set() + relpaths = [] + conditions = {} + known_unconditional_annotations = ["skip", "fail", "asserts", "random"] + known_conditional_annotations = [ + "skip-if", + "fail-if", + "run-if", + "fails-if", + "fuzzy-if", + "random-if", + "asserts-if", + ] + for t in tests: + relpath = t.get("srcdir_relpath") + relpaths.append(relpath) + reader = self.build_obj.mozbuild_reader(config_mode="empty") + files_info = reader.files_info(relpaths) + for t in tests: + if not matches_filters(t): + continue + if "referenced-test" in t: + # Avoid double-counting reftests: disregard reference file entries + continue + if show_annotations: + for key in t: + if key in known_unconditional_annotations: + annotation_count += 1 + if key in known_conditional_annotations: + annotation_count += 1 + # Here 'key' is a manifest annotation type like 'skip-if' and t[key] + # is the associated condition. For example, the manifestparser + # manifest annotation, "skip-if = os == 'win'", is expected to be + # encoded as t['skip-if'] = "os == 'win'". + # To allow for reftest manifests, t[key] may have multiple entries + # separated by ';', each corresponding to a condition for that test + # and annotation type. For example, + # "skip-if(Android&&webrender) skip-if(OSX)", would be + # encoded as t['skip-if'] = "Android&&webrender;OSX". + annotation_conditions = t[key].split(";") + + # if key has \n in it, we need to strip it. for manifestparser format + # 1) from the beginning of the line + # 2) different conditions if in the middle of the line + annotation_conditions = [ + x.strip("\n") for x in annotation_conditions + ] + temp = [] + for condition in annotation_conditions: + temp.extend(condition.split("\n")) + annotation_conditions = temp + + for condition in annotation_conditions: + condition_count += 1 + # Trim reftest fuzzy-if ranges: everything after the first comma + # eg. "Android,0-2,1-3" -> "Android" + condition = condition.split(",")[0] + if condition not in conditions: + conditions[condition] = 0 + conditions[condition] += 1 + test_count += 1 + relpath = t.get("srcdir_relpath") + if relpath in files_info: + bug_component = files_info[relpath].get("BUG_COMPONENT") + if bug_component: + key = "{}::{}".format( + bug_component.product, bug_component.component + ) + else: + key = "<unknown bug component>" + if (not components) or (key in components): + component_set.add(key) + test_info = {"test": relpath} + for test_key in display_keys: + value = t.get(test_key) + if value: + test_info[test_key] = value + if t.get("fail-if"): + failed_count += 1 + if t.get("fails-if"): + failed_count += 1 + if t.get("skip-if"): + skipped_count += 1 + + if "manifest_relpath" in t and "manifest" in t: + if "web-platform" in t["manifest_relpath"]: + test_info["manifest"] = [t["manifest"]] + else: + test_info["manifest"] = [t["manifest_relpath"]] + + # handle included manifests as ancestor:child + if t.get("ancestor_manifest", None): + test_info["manifest"] = [ + "%s:%s" + % (t["ancestor_manifest"], test_info["manifest"][0]) + ] + + # add in intermittent failure data + if ifd.get(relpath): + if_data = ifd.get(relpath) + test_info["failure_count"] = if_data["count"] + if show_testruns: + total_runs = 0 + for m in test_info["manifest"]: + if m in runcount.keys(): + for x in runcount.get(m, []): + if not x: + break + total_runs += x[3] + if total_runs > 0: + test_info["total_runs"] = total_runs + + if show_tests: + rkey = key if show_components else "all" + if rkey in by_component["tests"]: + # Avoid duplicates: Some test paths have multiple TestResolver + # entries, as when a test is included by multiple manifests. + found = False + for ctest in by_component["tests"][rkey]: + if ctest["test"] == test_info["test"]: + found = True + break + if not found: + by_component["tests"][rkey].append(test_info) + else: + for ti in by_component["tests"][rkey]: + if ti["test"] == test_info["test"]: + if ( + test_info["manifest"][0] + not in ti["manifest"] + ): + ti_manifest = test_info["manifest"] + if test_info.get( + "ancestor_manifest", None + ): + ti_manifest = "%s:%s" % ( + test_info["ancestor_manifest"], + ti_manifest, + ) + ti["manifest"].extend(ti_manifest) + else: + by_component["tests"][rkey] = [test_info] + if show_tests: + for key in by_component["tests"]: + by_component["tests"][key].sort(key=lambda k: k["test"]) + + by_component["description"] = self.description( + components, + flavor, + subsuite, + paths, + show_manifests, + show_tests, + show_summary, + show_annotations, + filter_values, + filter_keys, + start, + end, + ) + + if show_summary: + by_component["summary"] = {} + by_component["summary"]["components"] = len(component_set) + by_component["summary"]["manifests"] = manifest_count + by_component["summary"]["tests"] = test_count + by_component["summary"]["failed tests"] = failed_count + by_component["summary"]["skipped tests"] = skipped_count + + if show_annotations: + by_component["annotations"] = {} + by_component["annotations"]["total annotations"] = annotation_count + by_component["annotations"]["total conditions"] = condition_count + by_component["annotations"]["unique conditions"] = len(conditions) + by_component["annotations"]["conditions"] = conditions + + self.write_report(by_component, output_file) + + end_time = datetime.datetime.now() + self.log_verbose( + "%d seconds total to generate report" + % (end_time - start_time).total_seconds() + ) + + def write_report(self, by_component, output_file): + json_report = json.dumps(by_component, indent=2, sort_keys=True) + if output_file: + output_file = os.path.abspath(output_file) + output_dir = os.path.dirname(output_file) + if not os.path.isdir(output_dir): + os.makedirs(output_dir) + + with open(output_file, "w") as f: + f.write(json_report) + else: + print(json_report) + + def report_diff(self, before, after, output_file): + """ + Support for 'mach test-info report-diff'. + """ + + def get_file(path_or_url): + if urlparse.urlparse(path_or_url).scheme: + response = requests.get(path_or_url) + response.raise_for_status() + return json.loads(response.text) + with open(path_or_url) as f: + return json.load(f) + + report1 = get_file(before) + report2 = get_file(after) + + by_component = {"tests": {}, "summary": {}} + self.diff_summaries(by_component, report1["summary"], report2["summary"]) + self.diff_all_components(by_component, report1["tests"], report2["tests"]) + self.write_report(by_component, output_file) + + def diff_summaries(self, by_component, summary1, summary2): + """ + Update by_component with comparison of summaries. + """ + all_keys = set(summary1.keys()) | set(summary2.keys()) + for key in all_keys: + delta = summary2.get(key, 0) - summary1.get(key, 0) + by_component["summary"]["%s delta" % key] = delta + + def diff_all_components(self, by_component, tests1, tests2): + """ + Update by_component with any added/deleted tests, for all components. + """ + self.added_count = 0 + self.deleted_count = 0 + for component in tests1: + component1 = tests1[component] + component2 = [] if component not in tests2 else tests2[component] + self.diff_component(by_component, component, component1, component2) + for component in tests2: + if component not in tests1: + component2 = tests2[component] + self.diff_component(by_component, component, [], component2) + by_component["summary"]["added tests"] = self.added_count + by_component["summary"]["deleted tests"] = self.deleted_count + + def diff_component(self, by_component, component, component1, component2): + """ + Update by_component[component] with any added/deleted tests for the + named component. + "added": tests found in component2 but missing from component1. + "deleted": tests found in component1 but missing from component2. + """ + tests1 = set([t["test"] for t in component1]) + tests2 = set([t["test"] for t in component2]) + deleted = tests1 - tests2 + added = tests2 - tests1 + if deleted or added: + by_component["tests"][component] = {} + if deleted: + by_component["tests"][component]["deleted"] = sorted(list(deleted)) + if added: + by_component["tests"][component]["added"] = sorted(list(added)) + self.added_count += len(added) + self.deleted_count += len(deleted) + common = len(tests1.intersection(tests2)) + self.log_verbose( + "%s: %d deleted, %d added, %d common" + % (component, len(deleted), len(added), common) + ) |