summaryrefslogtreecommitdiffstats
path: root/testing/runcppunittests.py
diff options
context:
space:
mode:
Diffstat (limited to 'testing/runcppunittests.py')
-rwxr-xr-xtesting/runcppunittests.py387
1 files changed, 387 insertions, 0 deletions
diff --git a/testing/runcppunittests.py b/testing/runcppunittests.py
new file mode 100755
index 0000000000..9924a1dbeb
--- /dev/null
+++ b/testing/runcppunittests.py
@@ -0,0 +1,387 @@
+#!/usr/bin/env python
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+from optparse import OptionParser
+from os import environ as env
+
+import manifestparser
+import mozcrash
+import mozfile
+import mozinfo
+import mozlog
+import mozprocess
+import mozrunner.utils
+
+SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
+
+# Export directory js/src for tests that need it.
+env["CPP_UNIT_TESTS_DIR_JS_SRC"] = os.path.abspath(os.path.join(SCRIPT_DIR, "..", ".."))
+
+
+class CPPUnitTests(object):
+ # Time (seconds) to wait for test process to complete
+ TEST_PROC_TIMEOUT = 900
+ # Time (seconds) in which process will be killed if it produces no output.
+ TEST_PROC_NO_OUTPUT_TIMEOUT = 300
+
+ def run_one_test(
+ self,
+ prog,
+ env,
+ symbols_path=None,
+ utility_path=None,
+ interactive=False,
+ timeout_factor=1,
+ ):
+ """
+ Run a single C++ unit test program.
+
+ Arguments:
+ * prog: The path to the test program to run.
+ * env: The environment to use for running the program.
+ * symbols_path: A path to a directory containing Breakpad-formatted
+ symbol files for producing stack traces on crash.
+ * timeout_factor: An optional test-specific timeout multiplier.
+
+ Return True if the program exits with a zero status, False otherwise.
+ """
+ CPPUnitTests.run_one_test.timed_out = False
+ output = []
+
+ def timeout_handler(proc):
+ CPPUnitTests.run_one_test.timed_out = True
+ message = "timed out after %d seconds" % CPPUnitTests.TEST_PROC_TIMEOUT
+ self.log.test_end(
+ basename, status="TIMEOUT", expected="PASS", message=message
+ )
+ mozcrash.kill_and_get_minidump(proc.pid, tempdir, utility_path)
+
+ def output_timeout_handler(proc):
+ CPPUnitTests.run_one_test.timed_out = True
+ message = (
+ "timed out after %d seconds without output"
+ % CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT
+ )
+ self.log.test_end(
+ basename, status="TIMEOUT", expected="PASS", message=message
+ )
+ mozcrash.kill_and_get_minidump(proc.pid, tempdir, utility_path)
+
+ def output_line_handler(_, line):
+ fixed_line = self.fix_stack(line) if self.fix_stack else line
+ if interactive:
+ print(fixed_line)
+ else:
+ output.append(fixed_line)
+
+ basename = os.path.basename(prog)
+ self.log.test_start(basename)
+ with mozfile.TemporaryDirectory() as tempdir:
+ test_timeout = CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
+ proc = mozprocess.run_and_wait(
+ [prog],
+ cwd=tempdir,
+ env=env,
+ output_line_handler=output_line_handler,
+ timeout=test_timeout,
+ timeout_handler=timeout_handler,
+ output_timeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT,
+ output_timeout_handler=output_timeout_handler,
+ )
+
+ if output:
+ output = "\n%s" % "\n".join(output)
+ self.log.process_output(proc.pid, output, command=[prog])
+ if CPPUnitTests.run_one_test.timed_out:
+ return False
+ if mozcrash.check_for_crashes(tempdir, symbols_path, test_name=basename):
+ self.log.test_end(basename, status="CRASH", expected="PASS")
+ return False
+ result = proc.returncode == 0
+ if not result:
+ self.log.test_end(
+ basename,
+ status="FAIL",
+ expected="PASS",
+ message=("test failed with return code %d" % proc.returncode),
+ )
+ else:
+ self.log.test_end(basename, status="PASS", expected="PASS")
+ return result
+
+ def build_core_environment(self, env):
+ """
+ Add environment variables likely to be used across all platforms, including remote systems.
+ """
+ env["MOZ_XRE_DIR"] = self.xre_path
+ # TODO: switch this to just abort once all C++ unit tests have
+ # been fixed to enable crash reporting
+ env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
+ env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
+ env["MOZ_CRASHREPORTER"] = "1"
+ return env
+
+ def build_environment(self):
+ """
+ Create and return a dictionary of all the appropriate env variables and values.
+ On a remote system, we overload this to set different values and are missing things
+ like os.environ and PATH.
+ """
+ if not os.path.isdir(self.xre_path):
+ raise Exception("xre_path does not exist: %s", self.xre_path)
+ env = dict(os.environ)
+ env = self.build_core_environment(env)
+ pathvar = ""
+ libpath = self.xre_path
+ if mozinfo.os == "linux":
+ pathvar = "LD_LIBRARY_PATH"
+ elif mozinfo.os == "mac":
+ applibpath = os.path.join(os.path.dirname(libpath), "MacOS")
+ if os.path.exists(applibpath):
+ # Set the library load path to Contents/MacOS if we're run from
+ # the app bundle.
+ libpath = applibpath
+ pathvar = "DYLD_LIBRARY_PATH"
+ elif mozinfo.os == "win":
+ pathvar = "PATH"
+ if pathvar:
+ if pathvar in env:
+ env[pathvar] = "%s%s%s" % (libpath, os.pathsep, env[pathvar])
+ else:
+ env[pathvar] = libpath
+
+ symbolizer_path = None
+ if mozinfo.info["asan"]:
+ symbolizer_path = "ASAN_SYMBOLIZER_PATH"
+ elif mozinfo.info["tsan"]:
+ symbolizer_path = "TSAN_SYMBOLIZER_PATH"
+
+ if symbolizer_path is not None:
+ # Use llvm-symbolizer for ASan/TSan if available/required
+ if symbolizer_path in env and os.path.isfile(env[symbolizer_path]):
+ llvmsym = env[symbolizer_path]
+ else:
+ llvmsym = os.path.join(
+ self.xre_path, "llvm-symbolizer" + mozinfo.info["bin_suffix"]
+ )
+ if os.path.isfile(llvmsym):
+ env[symbolizer_path] = llvmsym
+ self.log.info("Using LLVM symbolizer at %s" % llvmsym)
+ else:
+ self.log.info("Failed to find LLVM symbolizer at %s" % llvmsym)
+
+ return env
+
+ def run_tests(
+ self,
+ programs,
+ xre_path,
+ symbols_path=None,
+ utility_path=None,
+ interactive=False,
+ ):
+ """
+ Run a set of C++ unit test programs.
+
+ Arguments:
+ * programs: An iterable containing (test path, test timeout factor) tuples
+ * xre_path: A path to a directory containing a XUL Runtime Environment.
+ * symbols_path: A path to a directory containing Breakpad-formatted
+ symbol files for producing stack traces on crash.
+ * utility_path: A path to a directory containing utility programs
+ (xpcshell et al)
+
+ Returns True if all test programs exited with a zero status, False
+ otherwise.
+ """
+ self.xre_path = xre_path
+ self.log = mozlog.get_default_logger()
+ if utility_path:
+ self.fix_stack = mozrunner.utils.get_stack_fixer_function(
+ utility_path, symbols_path
+ )
+ self.log.suite_start(programs, name="cppunittest")
+ env = self.build_environment()
+ pass_count = 0
+ fail_count = 0
+ for prog in programs:
+ test_path = prog[0]
+ timeout_factor = prog[1]
+ single_result = self.run_one_test(
+ test_path, env, symbols_path, utility_path, interactive, timeout_factor
+ )
+ if single_result:
+ pass_count += 1
+ else:
+ fail_count += 1
+ self.log.suite_end()
+
+ # Mozharness-parseable summary formatting.
+ self.log.info("Result summary:")
+ self.log.info("cppunittests INFO | Passed: %d" % pass_count)
+ self.log.info("cppunittests INFO | Failed: %d" % fail_count)
+ return fail_count == 0
+
+
+class CPPUnittestOptions(OptionParser):
+ def __init__(self):
+ OptionParser.__init__(self)
+ self.add_option(
+ "--xre-path",
+ action="store",
+ type="string",
+ dest="xre_path",
+ default=None,
+ help="absolute path to directory containing XRE (probably xulrunner)",
+ )
+ self.add_option(
+ "--symbols-path",
+ action="store",
+ type="string",
+ dest="symbols_path",
+ default=None,
+ help="absolute path to directory containing breakpad symbols, or "
+ "the URL of a zip file containing symbols",
+ )
+ self.add_option(
+ "--manifest-path",
+ action="store",
+ type="string",
+ dest="manifest_path",
+ default=None,
+ help="path to test manifest, if different from the path to test binaries",
+ )
+ self.add_option(
+ "--utility-path",
+ action="store",
+ type="string",
+ dest="utility_path",
+ default=None,
+ help="path to directory containing utility programs",
+ )
+
+
+def extract_unittests_from_args(args, environ, manifest_path):
+ """Extract unittests from args, expanding directories as needed"""
+ mp = manifestparser.TestManifest(strict=True)
+ tests = []
+ binary_path = None
+
+ if manifest_path:
+ mp.read(manifest_path)
+ binary_path = os.path.abspath(args[0])
+ else:
+ for p in args:
+ if os.path.isdir(p):
+ try:
+ mp.read(os.path.join(p, "cppunittest.toml"))
+ except IOError:
+ files = [os.path.abspath(os.path.join(p, x)) for x in os.listdir(p)]
+ tests.extend(
+ (f, 1) for f in files if os.access(f, os.R_OK | os.X_OK)
+ )
+ else:
+ tests.append((os.path.abspath(p), 1))
+
+ # We don't use the manifest parser's existence-check only because it will
+ # fail on Windows due to the `.exe` suffix.
+ active_tests = mp.active_tests(exists=False, disabled=False, **environ)
+ suffix = ".exe" if mozinfo.isWin else ""
+ if binary_path:
+ tests.extend(
+ [
+ (
+ os.path.join(binary_path, test["relpath"] + suffix),
+ int(test.get("requesttimeoutfactor", 1)),
+ )
+ for test in active_tests
+ ]
+ )
+ else:
+ tests.extend(
+ [
+ (test["path"] + suffix, int(test.get("requesttimeoutfactor", 1)))
+ for test in active_tests
+ ]
+ )
+
+ # Manually confirm that all tests named in the manifest exist.
+ errors = False
+ log = mozlog.get_default_logger()
+ for test in tests:
+ if not os.path.isfile(test[0]):
+ errors = True
+ log.error("test file not found: %s" % test[0])
+
+ if errors:
+ raise RuntimeError("One or more cppunittests not found; aborting.")
+
+ return tests
+
+
+def update_mozinfo():
+ """walk up directories to find mozinfo.json update the info"""
+ path = SCRIPT_DIR
+ dirs = set()
+ while path != os.path.expanduser("~"):
+ if path in dirs:
+ break
+ dirs.add(path)
+ path = os.path.split(path)[0]
+ mozinfo.find_and_update_from_json(*dirs)
+
+
+def run_test_harness(options, args):
+ update_mozinfo()
+ progs = extract_unittests_from_args(args, mozinfo.info, options.manifest_path)
+ options.xre_path = os.path.abspath(options.xre_path)
+ options.utility_path = os.path.abspath(options.utility_path)
+ tester = CPPUnitTests()
+ result = tester.run_tests(
+ progs,
+ options.xre_path,
+ options.symbols_path,
+ options.utility_path,
+ )
+
+ return result
+
+
+def main():
+ parser = CPPUnittestOptions()
+ mozlog.commandline.add_logging_group(parser)
+ options, args = parser.parse_args()
+ if not args:
+ print(
+ """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0],
+ file=sys.stderr,
+ )
+ sys.exit(1)
+ if not options.xre_path:
+ print("""Error: --xre-path is required""", file=sys.stderr)
+ sys.exit(1)
+ if options.manifest_path and len(args) > 1:
+ print(
+ "Error: multiple arguments not supported with --test-manifest",
+ file=sys.stderr,
+ )
+ sys.exit(1)
+ log = mozlog.commandline.setup_logging(
+ "cppunittests", options, {"tbpl": sys.stdout}
+ )
+ try:
+ result = run_test_harness(options, args)
+ except Exception as e:
+ log.error(str(e))
+ result = False
+
+ sys.exit(0 if result else 1)
+
+
+if __name__ == "__main__":
+ main()