import math import mozinfo class Bisect(object): "Class for creating, bisecting and summarizing for --bisect-chunk option." def __init__(self, harness): super(Bisect, self).__init__() self.summary = [] self.contents = {} self.repeat = 10 self.failcount = 0 self.max_failures = 3 def setup(self, tests): """This method is used to initialize various variables that are required for test bisection""" status = 0 self.contents.clear() # We need totalTests key in contents for sanity check self.contents["totalTests"] = tests self.contents["tests"] = tests self.contents["loop"] = 0 return status def reset(self, expectedError, result): """This method is used to initialize self.expectedError and self.result for each loop in runtests.""" self.expectedError = expectedError self.result = result def get_tests_for_bisection(self, options, tests): """Make a list of tests for bisection from a given list of tests""" bisectlist = [] for test in tests: bisectlist.append(test) if test.endswith(options.bisectChunk): break return bisectlist def pre_test(self, options, tests, status): """This method is used to call other methods for setting up variables and getting the list of tests for bisection.""" if options.bisectChunk == "default": return tests # The second condition in 'if' is required to verify that the failing # test is the last one. elif "loop" not in self.contents or not self.contents["tests"][-1].endswith( options.bisectChunk ): tests = self.get_tests_for_bisection(options, tests) status = self.setup(tests) return self.next_chunk_binary(options, status) def post_test(self, options, expectedError, result): """This method is used to call other methods to summarize results and check whether a sanity check is done or not.""" self.reset(expectedError, result) status = self.summarize_chunk(options) # Check whether sanity check has to be done. Also it is necessary to check whether # options.bisectChunk is present in self.expectedError as we do not want to run # if it is "default". if status == -1 and options.bisectChunk in self.expectedError: # In case we have a debug build, we don't want to run a sanity # check, will take too much time. if mozinfo.info["debug"]: return status testBleedThrough = self.contents["testsToRun"][0] tests = self.contents["totalTests"] tests.remove(testBleedThrough) # To make sure that the failing test is dependent on some other # test. if options.bisectChunk in testBleedThrough: return status status = self.setup(tests) self.summary.append("Sanity Check:") return status def next_chunk_reverse(self, options, status): "This method is used to bisect the tests in a reverse search fashion." # Base Cases. if self.contents["loop"] <= 1: self.contents["testsToRun"] = self.contents["tests"] if self.contents["loop"] == 1: self.contents["testsToRun"] = [self.contents["tests"][-1]] self.contents["loop"] += 1 return self.contents["testsToRun"] if "result" in self.contents: if self.contents["result"] == "PASS": chunkSize = self.contents["end"] - self.contents["start"] self.contents["end"] = self.contents["start"] - 1 self.contents["start"] = self.contents["end"] - chunkSize # self.contents['result'] will be expected error only if it fails. elif self.contents["result"] == "FAIL": self.contents["tests"] = self.contents["testsToRun"] status = 1 # for initializing # initialize if status: totalTests = len(self.contents["tests"]) chunkSize = int(math.ceil(totalTests / 10.0)) self.contents["start"] = totalTests - chunkSize - 1 self.contents["end"] = totalTests - 2 start = self.contents["start"] end = self.contents["end"] + 1 self.contents["testsToRun"] = self.contents["tests"][start:end] self.contents["testsToRun"].append(self.contents["tests"][-1]) self.contents["loop"] += 1 return self.contents["testsToRun"] def next_chunk_binary(self, options, status): "This method is used to bisect the tests in a binary search fashion." # Base cases. if self.contents["loop"] <= 1: self.contents["testsToRun"] = self.contents["tests"] if self.contents["loop"] == 1: self.contents["testsToRun"] = [self.contents["tests"][-1]] self.contents["loop"] += 1 return self.contents["testsToRun"] # Initialize the contents dict. if status: totalTests = len(self.contents["tests"]) self.contents["start"] = 0 self.contents["end"] = totalTests - 2 # pylint --py3k W1619 mid = (self.contents["start"] + self.contents["end"]) / 2 if "result" in self.contents: if self.contents["result"] == "PASS": self.contents["end"] = mid elif self.contents["result"] == "FAIL": self.contents["start"] = mid + 1 mid = (self.contents["start"] + self.contents["end"]) / 2 start = mid + 1 end = self.contents["end"] + 1 self.contents["testsToRun"] = self.contents["tests"][start:end] if not self.contents["testsToRun"]: self.contents["testsToRun"].append(self.contents["tests"][mid]) self.contents["testsToRun"].append(self.contents["tests"][-1]) self.contents["loop"] += 1 return self.contents["testsToRun"] def summarize_chunk(self, options): "This method is used summarize the results after the list of tests is run." if options.bisectChunk == "default": # if no expectedError that means all the tests have successfully # passed. if len(self.expectedError) == 0: return -1 options.bisectChunk = self.expectedError.keys()[0] self.summary.append("\tFound Error in test: %s" % options.bisectChunk) return 0 # If options.bisectChunk is not in self.result then we need to move to # the next run. if options.bisectChunk not in self.result: return -1 self.summary.append("\tPass %d:" % self.contents["loop"]) if len(self.contents["testsToRun"]) > 1: self.summary.append( "\t\t%d test files(start,end,failing). [%s, %s, %s]" % ( len(self.contents["testsToRun"]), self.contents["testsToRun"][0], self.contents["testsToRun"][-2], self.contents["testsToRun"][-1], ) ) else: self.summary.append("\t\t1 test file [%s]" % self.contents["testsToRun"][0]) return self.check_for_intermittent(options) if self.result[options.bisectChunk] == "PASS": self.summary.append("\t\tno failures found.") if self.contents["loop"] == 1: status = -1 else: self.contents["result"] = "PASS" status = 0 elif self.result[options.bisectChunk] == "FAIL": if "expectedError" not in self.contents: self.summary.append("\t\t%s failed." % self.contents["testsToRun"][-1]) self.contents["expectedError"] = self.expectedError[options.bisectChunk] status = 0 elif ( self.expectedError[options.bisectChunk] == self.contents["expectedError"] ): self.summary.append( "\t\t%s failed with expected error." % self.contents["testsToRun"][-1] ) self.contents["result"] = "FAIL" status = 0 # This code checks for test-bleedthrough. Should work for any # algorithm. numberOfTests = len(self.contents["testsToRun"]) if numberOfTests < 3: # This means that only 2 tests are run. Since the last test # is the failing test itself therefore the bleedthrough # test is the first test self.summary.append( "TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the " "root cause for many of the above failures" % self.contents["testsToRun"][0] ) status = -1 else: self.summary.append( "\t\t%s failed with different error." % self.contents["testsToRun"][-1] ) status = -1 return status def check_for_intermittent(self, options): "This method is used to check whether a test is an intermittent." if self.result[options.bisectChunk] == "PASS": self.summary.append( "\t\tThe test %s passed." % self.contents["testsToRun"][0] ) if self.repeat > 0: # loop is set to 1 to again run the single test. self.contents["loop"] = 1 self.repeat -= 1 return 0 else: if self.failcount > 0: # -1 is being returned as the test is intermittent, so no need to bisect # further. return -1 # If the test does not fail even once, then proceed to next chunk for bisection. # loop is set to 2 to proceed on bisection. self.contents["loop"] = 2 return 1 elif self.result[options.bisectChunk] == "FAIL": self.summary.append( "\t\tThe test %s failed." % self.contents["testsToRun"][0] ) self.failcount += 1 self.contents["loop"] = 1 self.repeat -= 1 # self.max_failures is the maximum number of times a test is allowed # to fail to be called an intermittent. If a test fails more than # limit set, it is a perma-fail. if self.failcount < self.max_failures: if self.repeat == 0: # -1 is being returned as the test is intermittent, so no need to bisect # further. return -1 return 0 else: self.summary.append( "TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the " "root cause for many of the above failures" % self.contents["testsToRun"][0] ) return -1 def print_summary(self): "This method is used to print the recorded summary." print("Bisection summary:") for line in self.summary: print(line)