diff options
Diffstat (limited to '')
| -rw-r--r-- | testing/mozbase/mozcrash/mozcrash/__init__.py | 9 | ||||
| -rw-r--r-- | testing/mozbase/mozcrash/mozcrash/mozcrash.py | 816 | ||||
| -rw-r--r-- | testing/mozbase/mozcrash/setup.cfg | 2 | ||||
| -rw-r--r-- | testing/mozbase/mozcrash/setup.py | 33 | ||||
| -rw-r--r-- | testing/mozbase/mozcrash/tests/conftest.py | 127 | ||||
| -rw-r--r-- | testing/mozbase/mozcrash/tests/manifest.ini | 7 | ||||
| -rw-r--r-- | testing/mozbase/mozcrash/tests/test_basic.py | 43 | ||||
| -rw-r--r-- | testing/mozbase/mozcrash/tests/test_java_exception.py | 51 | ||||
| -rw-r--r-- | testing/mozbase/mozcrash/tests/test_save_path.py | 68 | ||||
| -rw-r--r-- | testing/mozbase/mozcrash/tests/test_stackwalk.py | 42 | ||||
| -rw-r--r-- | testing/mozbase/mozcrash/tests/test_symbols_path.py | 97 |
11 files changed, 1295 insertions, 0 deletions
diff --git a/testing/mozbase/mozcrash/mozcrash/__init__.py b/testing/mozbase/mozcrash/mozcrash/__init__.py new file mode 100644 index 0000000000..a6dfab2b24 --- /dev/null +++ b/testing/mozbase/mozcrash/mozcrash/__init__.py @@ -0,0 +1,9 @@ +# flake8: noqa +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +""" +mozcrash is a library for getting a stack trace out of processes that have crashed +and left behind a minidump file using the Google Breakpad library. +""" +from .mozcrash import * diff --git a/testing/mozbase/mozcrash/mozcrash/mozcrash.py b/testing/mozbase/mozcrash/mozcrash/mozcrash.py new file mode 100644 index 0000000000..dc6fac560d --- /dev/null +++ b/testing/mozbase/mozcrash/mozcrash/mozcrash.py @@ -0,0 +1,816 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import glob +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import tempfile +import traceback +import zipfile +from collections import namedtuple + +import mozfile +import mozinfo +import mozlog +import six +from redo import retriable + +__all__ = [ + "check_for_crashes", + "check_for_java_exception", + "kill_and_get_minidump", + "log_crashes", + "cleanup_pending_crash_reports", +] + + +StackInfo = namedtuple( + "StackInfo", + [ + "minidump_path", + "signature", + "stackwalk_stdout", + "stackwalk_stderr", + "stackwalk_retcode", + "stackwalk_errors", + "extra", + "reason", + "java_stack", + ], +) + + +def get_logger(): + structured_logger = mozlog.get_default_logger("mozcrash") + if structured_logger is None: + return mozlog.unstructured.getLogger("mozcrash") + return structured_logger + + +def check_for_crashes( + dump_directory, + symbols_path=None, + stackwalk_binary=None, + dump_save_path=None, + test_name=None, + quiet=False, + keep=False, +): + """ + Print a stack trace for minidump files left behind by a crashing program. + + `dump_directory` will be searched for minidump files. Any minidump files found will + have `stackwalk_binary` executed on them, with `symbols_path` passed as an extra + argument. + + `stackwalk_binary` should be a path to the minidump-stackwalk binary. + If `stackwalk_binary` is not set, the MINIDUMP_STACKWALK environment variable + will be checked and its value used if it is not empty. If neither is set, then + ~/.mozbuild/minidump-stackwalk/minidump-stackwalk will be used. + + `symbols_path` should be a path to a directory containing symbols to use for + dump processing. This can either be a path to a directory containing Breakpad-format + symbols, or a URL to a zip file containing a set of symbols. + + If `dump_save_path` is set, it should be a path to a directory in which to copy minidump + files for safekeeping after a stack trace has been printed. If not set, the environment + variable MINIDUMP_SAVE_PATH will be checked and its value used if it is not empty. + + If `test_name` is set it will be used as the test name in log output. If not set the + filename of the calling function will be used. + + If `quiet` is set, no PROCESS-CRASH message will be printed to stdout if a + crash is detected. + + If `keep` is set, minidump files will not be removed after processing. + + Returns number of minidump files found. + """ + + # try to get the caller's filename if no test name is given + if test_name is None: + try: + test_name = os.path.basename(sys._getframe(1).f_code.co_filename) + except Exception: + test_name = "unknown" + + if not quiet: + print("mozcrash checking %s for minidumps..." % dump_directory) + + crash_info = CrashInfo( + dump_directory, + symbols_path, + dump_save_path=dump_save_path, + stackwalk_binary=stackwalk_binary, + keep=keep, + ) + + crash_count = 0 + for info in crash_info: + crash_count += 1 + output = None + if info.java_stack: + output = "PROCESS-CRASH | {name} | {stack}".format( + name=test_name, stack=info.java_stack + ) + elif not quiet: + stackwalk_output = ["Crash dump filename: {}".format(info.minidump_path)] + if info.reason: + stackwalk_output.append("Mozilla crash reason: %s" % info.reason) + if info.stackwalk_stderr: + stackwalk_output.append("stderr from minidump-stackwalk:") + stackwalk_output.append(info.stackwalk_stderr) + elif info.stackwalk_stdout is not None: + stackwalk_output.append(info.stackwalk_stdout) + if info.stackwalk_retcode is not None and info.stackwalk_retcode != 0: + stackwalk_output.append( + "minidump-stackwalk exited with return code {}".format( + info.stackwalk_retcode + ) + ) + signature = info.signature if info.signature else "unknown top frame" + + output = "PROCESS-CRASH | {reason} [{sig}] | {name}\n{out}\n{err}".format( + reason=info.reason, + name=test_name, + sig=signature, + out="\n".join(stackwalk_output), + err="\n".join(info.stackwalk_errors), + ) + if output is not None: + if six.PY2 and sys.stdout.encoding != "UTF-8": + output = output.encode("utf-8") + print(output) + + return crash_count + + +def log_crashes( + logger, + dump_directory, + symbols_path, + process=None, + test=None, + stackwalk_binary=None, + dump_save_path=None, + quiet=False, +): + """Log crashes using a structured logger""" + crash_count = 0 + for info in CrashInfo( + dump_directory, + symbols_path, + dump_save_path=dump_save_path, + stackwalk_binary=stackwalk_binary, + ): + crash_count += 1 + if not quiet: + kwargs = info._asdict() + kwargs.pop("extra") + logger.crash(process=process, test=test, **kwargs) + return crash_count + + +# Function signatures of abort functions which should be ignored when +# determining the appropriate frame for the crash signature. +ABORT_SIGNATURES = ( + "Abort(char const*)", + "RustMozCrash", + "NS_DebugBreak", + # This signature is part of Rust panic stacks on some platforms. On + # others, it includes a template parameter containing "core::panic::" and + # is automatically filtered out by that pattern. + "core::ops::function::Fn::call", + "gkrust_shared::panic_hook", + "mozglue_static::panic_hook", + "intentional_panic", + "mozalloc_abort", + "mozalloc_abort(char const* const)", + "static void Abort(const char *)", + "std::sys_common::backtrace::__rust_end_short_backtrace", + "rust_begin_unwind", + # This started showing up when we enabled dumping inlined functions + "MOZ_Crash(char const*, int, char const*)", + "<alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call", +) + +# Similar to above, but matches if the substring appears anywhere in the +# frame's signature. +ABORT_SUBSTRINGS = ( + # On some platforms, Rust panic frames unfortunately appear without the + # std::panicking or core::panic namespaces. + "_panic_", + "core::panic::", + "core::panicking::", + "core::result::unwrap_failed", + "std::panicking::", +) + + +class CrashInfo(object): + """Get information about a crash based on dump files. + + Typical usage is to iterate over the CrashInfo object. This returns StackInfo + objects, one for each crash dump file that is found in the dump_directory. + + :param dump_directory: Path to search for minidump files + :param symbols_path: Path to a path to a directory containing symbols to use for + dump processing. This can either be a path to a directory + containing Breakpad-format symbols, or a URL to a zip file + containing a set of symbols. + :param dump_save_path: Path to which to save the dump files. If this is None, + the MINIDUMP_SAVE_PATH environment variable will be used. + :param stackwalk_binary: Path to the minidump-stackwalk binary. If this is None, + the MINIDUMP_STACKWALK environment variable will be used + as the path to the minidump binary. If neither is set, + then ~/.mozbuild/minidump-stackwalk/minidump-stackwalk + will be used.""" + + def __init__( + self, + dump_directory, + symbols_path, + dump_save_path=None, + stackwalk_binary=None, + keep=False, + ): + self.dump_directory = dump_directory + self.symbols_path = symbols_path + self.remove_symbols = False + self.brief_output = False + self.keep = keep + + if dump_save_path is None: + dump_save_path = os.environ.get("MINIDUMP_SAVE_PATH", None) + self.dump_save_path = dump_save_path + + if stackwalk_binary is None: + stackwalk_binary = os.environ.get("MINIDUMP_STACKWALK", None) + if stackwalk_binary is None: + # Location of minidump-stackwalk installed by "mach bootstrap". + executable_name = "minidump-stackwalk" + state_dir = os.environ.get( + "MOZBUILD_STATE_PATH", + os.path.expanduser(os.path.join("~", ".mozbuild")), + ) + stackwalk_binary = os.path.join(state_dir, executable_name, executable_name) + if mozinfo.isWin and not stackwalk_binary.endswith(".exe"): + stackwalk_binary += ".exe" + if os.path.exists(stackwalk_binary): + # If we reach this point, then we're almost certainly + # running on a local user's machine. Full minidump-stackwalk + # output is a bit noisy and verbose for that use-case, + # so we should use the --brief output. + self.brief_output = True + + self.stackwalk_binary = stackwalk_binary + + self.logger = get_logger() + self._dump_files = None + + @retriable(attempts=5, sleeptime=5, sleepscale=2) + def _get_symbols(self): + if not self.symbols_path: + self.logger.warning( + "No local symbols_path provided, only http symbols will be used." + ) + + # This updates self.symbols_path so we only download once. + if mozfile.is_url(self.symbols_path): + self.remove_symbols = True + self.logger.info("Downloading symbols from: %s" % self.symbols_path) + # Get the symbols and write them to a temporary zipfile + data = six.moves.urllib.request.urlopen(self.symbols_path) + with tempfile.TemporaryFile() as symbols_file: + symbols_file.write(data.read()) + # extract symbols to a temporary directory (which we'll delete after + # processing all crashes) + self.symbols_path = tempfile.mkdtemp() + with zipfile.ZipFile(symbols_file, "r") as zfile: + mozfile.extract_zip(zfile, self.symbols_path) + + @property + def dump_files(self): + """List of tuple (path_to_dump_file, path_to_extra_file) for each dump + file in self.dump_directory. The extra files may not exist.""" + if self._dump_files is None: + self._dump_files = [ + (path, os.path.splitext(path)[0] + ".extra") + for path in reversed( + sorted(glob.glob(os.path.join(self.dump_directory, "*.dmp"))) + ) + ] + max_dumps = 10 + if len(self._dump_files) > max_dumps: + self.logger.warning( + "Found %d dump files -- limited to %d!" + % (len(self._dump_files), max_dumps) + ) + del self._dump_files[max_dumps:] + + return self._dump_files + + @property + def has_dumps(self): + """Boolean indicating whether any crash dump files were found in the + current directory""" + return len(self.dump_files) > 0 + + def __iter__(self): + for path, extra in self.dump_files: + rv = self._process_dump_file(path, extra) + yield rv + + if self.remove_symbols: + mozfile.remove(self.symbols_path) + + def _process_dump_file(self, path, extra): + """Process a single dump file using self.stackwalk_binary, and return a + tuple containing properties of the crash dump. + + :param path: Path to the minidump file to analyse + :return: A StackInfo tuple with the fields:: + minidump_path: Path of the dump file + signature: The top frame of the stack trace, or None if it + could not be determined. + stackwalk_stdout: String of stdout data from stackwalk + stackwalk_stderr: String of stderr data from stackwalk or + None if it succeeded + stackwalk_retcode: Return code from stackwalk + stackwalk_errors: List of errors in human-readable form that prevented + stackwalk being launched. + """ + self._get_symbols() + + errors = [] + signature = None + out = None + err = None + retcode = None + reason = None + java_stack = None + if ( + self.stackwalk_binary + and os.path.exists(self.stackwalk_binary) + and os.access(self.stackwalk_binary, os.X_OK) + ): + # Now build up the actual command + command = [self.stackwalk_binary] + + # Fallback to the symbols server for unknown symbols on automation + # (mostly for system libraries). + if "MOZ_AUTOMATION" in os.environ: + command.append("--symbols-url=https://symbols.mozilla.org/") + + with tempfile.TemporaryDirectory() as json_dir: + crash_id = os.path.basename(path)[:-4] + json_output = os.path.join(json_dir, "{}.trace".format(crash_id)) + # Specify the kind of output + command.append("--cyborg={}".format(json_output)) + if self.brief_output: + command.append("--brief") + + # The minidump path and symbols_path values are positional and come last + # (in practice the CLI parsers are more permissive, but best not to + # unecessarily play with fire). + command.append(path) + + if self.symbols_path: + command.append(self.symbols_path) + + self.logger.info("Copy/paste: {}".format(" ".join(command))) + # run minidump-stackwalk + p = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + (out, err) = p.communicate() + retcode = p.returncode + if six.PY3: + out = six.ensure_str(out) + err = six.ensure_str(err) + + if retcode == 0: + signature = self._generate_signature(json_output) + + # Strip parameters from signature + pmatch = re.search(r"(.*)\(.*\)", signature) + if pmatch: + signature = pmatch.group(1) + + else: + if not self.stackwalk_binary: + errors.append( + "MINIDUMP_STACKWALK not set, can't process dump. Either set " + "MINIDUMP_STACKWALK or use mach bootstrap --no-system-changes " + "to install minidump-stackwalk." + ) + elif self.stackwalk_binary and not os.path.exists(self.stackwalk_binary): + errors.append( + "MINIDUMP_STACKWALK binary not found: %s. Use mach bootstrap " + "--no-system-changes to install minidump-stackwalk." + % self.stackwalk_binary + ) + elif not os.access(self.stackwalk_binary, os.X_OK): + errors.append("This user cannot execute the MINIDUMP_STACKWALK binary.") + + if os.path.exists(extra): + crash_dict = self._parse_extra_file(extra) + reason = crash_dict.get("MozCrashReason") + java_stack = crash_dict.get("JavaStackTrace") + + if self.dump_save_path: + self._save_dump_file(path, extra) + + if os.path.exists(path) and not self.keep: + mozfile.remove(path) + if os.path.exists(extra) and not self.keep: + mozfile.remove(extra) + + return StackInfo( + path, + signature, + out, + err, + retcode, + errors, + extra, + reason, + java_stack, + ) + + def _generate_signature(self, json_path): + signature = None + + try: + json_file = open(json_path, "r") + crash_json = json.load(json_file) + json_file.close() + crashing_thread = crash_json.get("crashing_thread") or {} + frames = crashing_thread.get("frames") or [] + + flattened_frames = [] + for frame in frames: + for inline in frame.get("inlines") or []: + flattened_frames.append(inline.get("function")) + + flattened_frames.append( + frame.get("function") + or "{} + {}".format(frame.get("module"), frame.get("module_offset")) + ) + + for func in flattened_frames: + if not func: + continue + + signature = "@ %s" % func + + if not ( + func in ABORT_SIGNATURES + or any(pat in func for pat in ABORT_SUBSTRINGS) + ): + break + except Exception as e: + traceback.print_exc() + signature = "an error occurred while generating the signature: {}".format(e) + + return signature + + def _parse_extra_file(self, path): + with open(path) as file: + try: + return json.load(file) + except ValueError: + self.logger.warning(".extra file does not contain proper json") + return {} + + def _save_dump_file(self, path, extra): + if os.path.isfile(self.dump_save_path): + os.unlink(self.dump_save_path) + if not os.path.isdir(self.dump_save_path): + try: + os.makedirs(self.dump_save_path) + except OSError: + pass + + shutil.move(path, self.dump_save_path) + self.logger.info( + "Saved minidump as {}".format( + os.path.join(self.dump_save_path, os.path.basename(path)) + ) + ) + + if os.path.isfile(extra): + shutil.move(extra, self.dump_save_path) + self.logger.info( + "Saved app info as {}".format( + os.path.join(self.dump_save_path, os.path.basename(extra)) + ) + ) + + +def check_for_java_exception(logcat, test_name=None, quiet=False): + """ + Print a summary of a fatal Java exception, if present in the provided + logcat output. + + Today, exceptions in geckoview are usually noted in the minidump .extra file, allowing + java exceptions to be reported by the "normal" minidump processing, like log_crashes(); + therefore, this function may be extraneous (but maintained for now, while exception + handling is evolving). + + Example: + PROCESS-CRASH | <test-name> | java-exception java.lang.NullPointerException at org.mozilla.gecko.GeckoApp$21.run(GeckoApp.java:1833) # noqa + + `logcat` should be a list of strings. + + If `test_name` is set it will be used as the test name in log output. If not set the + filename of the calling function will be used. + + If `quiet` is set, no PROCESS-CRASH message will be printed to stdout if a + crash is detected. + + Returns True if a fatal Java exception was found, False otherwise. + """ + + # try to get the caller's filename if no test name is given + if test_name is None: + try: + test_name = os.path.basename(sys._getframe(1).f_code.co_filename) + except Exception: + test_name = "unknown" + + found_exception = False + + for i, line in enumerate(logcat): + # Logs will be of form: + # + # 01-30 20:15:41.937 E/GeckoAppShell( 1703): >>> REPORTING UNCAUGHT EXCEPTION FROM THREAD 9 ("GeckoBackgroundThread") # noqa + # 01-30 20:15:41.937 E/GeckoAppShell( 1703): java.lang.NullPointerException + # 01-30 20:15:41.937 E/GeckoAppShell( 1703): at org.mozilla.gecko.GeckoApp$21.run(GeckoApp.java:1833) # noqa + # 01-30 20:15:41.937 E/GeckoAppShell( 1703): at android.os.Handler.handleCallback(Handler.java:587) # noqa + if "REPORTING UNCAUGHT EXCEPTION" in line: + # Strip away the date, time, logcat tag and pid from the next two lines and + # concatenate the remainder to form a concise summary of the exception. + found_exception = True + if len(logcat) >= i + 3: + logre = re.compile(r".*\): \t?(.*)") + m = logre.search(logcat[i + 1]) + if m and m.group(1): + exception_type = m.group(1) + m = logre.search(logcat[i + 2]) + if m and m.group(1): + exception_location = m.group(1) + if not quiet: + output = ( + "PROCESS-CRASH | {name} | java-exception {type} {loc}".format( + name=test_name, type=exception_type, loc=exception_location + ) + ) + print(output.encode("utf-8")) + else: + print( + "Automation Error: java exception in logcat at line " + "{0} of {1}: {2}".format(i, len(logcat), line) + ) + break + + return found_exception + + +if mozinfo.isWin: + import ctypes + import uuid + + kernel32 = ctypes.windll.kernel32 + OpenProcess = kernel32.OpenProcess + CloseHandle = kernel32.CloseHandle + + def write_minidump(pid, dump_directory, utility_path): + """ + Write a minidump for a process. + + :param pid: PID of the process to write a minidump for. + :param dump_directory: Directory in which to write the minidump. + """ + PROCESS_QUERY_INFORMATION = 0x0400 + PROCESS_VM_READ = 0x0010 + GENERIC_READ = 0x80000000 + GENERIC_WRITE = 0x40000000 + CREATE_ALWAYS = 2 + FILE_ATTRIBUTE_NORMAL = 0x80 + INVALID_HANDLE_VALUE = -1 + + log = get_logger() + file_name = os.path.join(dump_directory, str(uuid.uuid4()) + ".dmp") + + if not os.path.exists(dump_directory): + # `kernal32.CreateFileW` can fail to create the dmp file if the dump + # directory was deleted or doesn't exist (error code 3). + os.makedirs(dump_directory) + + if mozinfo.info["bits"] != ctypes.sizeof(ctypes.c_voidp) * 8 and utility_path: + # We're not going to be able to write a minidump with ctypes if our + # python process was compiled for a different architecture than + # firefox, so we invoke the minidumpwriter utility program. + + minidumpwriter = os.path.normpath( + os.path.join(utility_path, "minidumpwriter.exe") + ) + log.info( + "Using {} to write a dump to {} for [{}]".format( + minidumpwriter, file_name, pid + ) + ) + if not os.path.exists(minidumpwriter): + log.error("minidumpwriter not found in {}".format(utility_path)) + return + + status = subprocess.Popen([minidumpwriter, str(pid), file_name]).wait() + if status: + log.error("minidumpwriter exited with status: %d" % status) + return + + log.info("Writing a dump to {} for [{}]".format(file_name, pid)) + + proc_handle = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, 0, pid) + if not proc_handle: + err = kernel32.GetLastError() + log.warning("unable to get handle for pid %d: %d" % (pid, err)) + return + + if not isinstance(file_name, six.text_type): + # Convert to unicode explicitly so our path will be valid as input + # to CreateFileW + file_name = six.text_type(file_name, sys.getfilesystemencoding()) + + file_handle = kernel32.CreateFileW( + file_name, + GENERIC_READ | GENERIC_WRITE, + 0, + None, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + None, + ) + if file_handle != INVALID_HANDLE_VALUE: + if not ctypes.windll.dbghelp.MiniDumpWriteDump( + proc_handle, + pid, + file_handle, + # Dump type - MiniDumpNormal + 0, + # Exception parameter + None, + # User stream parameter + None, + # Callback parameter + None, + ): + err = kernel32.GetLastError() + log.warning("unable to dump minidump file for pid %d: %d" % (pid, err)) + CloseHandle(file_handle) + else: + err = kernel32.GetLastError() + log.warning("unable to create minidump file for pid %d: %d" % (pid, err)) + CloseHandle(proc_handle) + + def kill_pid(pid): + """ + Terminate a process with extreme prejudice. + + :param pid: PID of the process to terminate. + """ + PROCESS_TERMINATE = 0x0001 + SYNCHRONIZE = 0x00100000 + WAIT_OBJECT_0 = 0x0 + WAIT_FAILED = -1 + logger = get_logger() + handle = OpenProcess(PROCESS_TERMINATE | SYNCHRONIZE, 0, pid) + if handle: + if kernel32.TerminateProcess(handle, 1): + # TerminateProcess is async; wait up to 30 seconds for process to + # actually terminate, then give up so that clients are not kept + # waiting indefinitely for hung processes. + status = kernel32.WaitForSingleObject(handle, 30000) + if status == WAIT_FAILED: + err = kernel32.GetLastError() + logger.warning( + "kill_pid(): wait failed (%d) terminating pid %d: error %d" + % (status, pid, err) + ) + elif status != WAIT_OBJECT_0: + logger.warning( + "kill_pid(): wait failed (%d) terminating pid %d" + % (status, pid) + ) + else: + err = kernel32.GetLastError() + logger.warning( + "kill_pid(): unable to terminate pid %d: %d" % (pid, err) + ) + CloseHandle(handle) + else: + err = kernel32.GetLastError() + logger.warning( + "kill_pid(): unable to get handle for pid %d: %d" % (pid, err) + ) + + +else: + + def kill_pid(pid): + """ + Terminate a process with extreme prejudice. + + :param pid: PID of the process to terminate. + """ + os.kill(pid, signal.SIGKILL) + + +def kill_and_get_minidump(pid, dump_directory, utility_path=None): + """ + Attempt to kill a process and leave behind a minidump describing its + execution state. + + :param pid: The PID of the process to kill. + :param dump_directory: The directory where a minidump should be written on + Windows, where the dump will be written from outside the process. + + On Windows a dump will be written using the MiniDumpWriteDump function + from DbgHelp.dll. On Linux and OS X the process will be sent a SIGABRT + signal to trigger minidump writing via a Breakpad signal handler. On other + platforms the process will simply be killed via SIGKILL. + + If the process is hung in such a way that it cannot respond to SIGABRT + it may still be running after this function returns. In that case it + is the caller's responsibility to deal with killing it. + """ + needs_killing = True + if mozinfo.isWin: + write_minidump(pid, dump_directory, utility_path) + elif mozinfo.isLinux or mozinfo.isMac: + os.kill(pid, signal.SIGABRT) + needs_killing = False + if needs_killing: + kill_pid(pid) + + +def cleanup_pending_crash_reports(): + """ + Delete any pending crash reports. + + The presence of pending crash reports may be reported by the browser, + affecting test results; it is best to ensure that these are removed + before starting any browser tests. + + Firefox stores pending crash reports in "<UAppData>/Crash Reports". + If the browser is not running, it cannot provide <UAppData>, so this + code tries to anticipate its value. + + See dom/system/OSFileConstants.cpp for platform variations of <UAppData>. + """ + if mozinfo.isWin: + location = os.path.expanduser( + "~\\AppData\\Roaming\\Mozilla\\Firefox\\Crash Reports" + ) + elif mozinfo.isMac: + location = os.path.expanduser( + "~/Library/Application Support/firefox/Crash Reports" + ) + else: + location = os.path.expanduser("~/.mozilla/firefox/Crash Reports") + logger = get_logger() + if os.path.exists(location): + try: + mozfile.remove(location) + logger.info("Removed pending crash reports at '%s'" % location) + except Exception: + pass + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--stackwalk-binary", "-b") + parser.add_argument("--dump-save-path", "-o") + parser.add_argument("--test-name", "-n") + parser.add_argument("--keep", action="store_true") + parser.add_argument("dump_directory") + parser.add_argument("symbols_path") + args = parser.parse_args() + + check_for_crashes( + args.dump_directory, + args.symbols_path, + stackwalk_binary=args.stackwalk_binary, + dump_save_path=args.dump_save_path, + test_name=args.test_name, + keep=args.keep, + ) diff --git a/testing/mozbase/mozcrash/setup.cfg b/testing/mozbase/mozcrash/setup.cfg new file mode 100644 index 0000000000..3c6e79cf31 --- /dev/null +++ b/testing/mozbase/mozcrash/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/testing/mozbase/mozcrash/setup.py b/testing/mozbase/mozcrash/setup.py new file mode 100644 index 0000000000..d67060149b --- /dev/null +++ b/testing/mozbase/mozcrash/setup.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from setuptools import setup + +PACKAGE_NAME = "mozcrash" +PACKAGE_VERSION = "2.2.0" + +# dependencies +deps = ["mozfile >= 1.0", "mozlog >= 6.0"] + +setup( + name=PACKAGE_NAME, + version=PACKAGE_VERSION, + description="Library for printing stack traces from minidumps " + "left behind by crashed processes", + long_description="see https://firefox-source-docs.mozilla.org/mozbase/index.html", + classifiers=[ + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.5", + ], + # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + keywords="mozilla", + author="Mozilla Automation and Tools team", + author_email="tools@lists.mozilla.org", + url="https://wiki.mozilla.org/Auto-tools/Projects/Mozbase", + license="MPL", + packages=["mozcrash"], + include_package_data=True, + zip_safe=False, + install_requires=deps, +) diff --git a/testing/mozbase/mozcrash/tests/conftest.py b/testing/mozbase/mozcrash/tests/conftest.py new file mode 100644 index 0000000000..4827515723 --- /dev/null +++ b/testing/mozbase/mozcrash/tests/conftest.py @@ -0,0 +1,127 @@ +# coding=UTF-8 + +import uuid + +import mozcrash +import pytest +from py._path.common import fspath + + +@pytest.fixture(scope="session") +def stackwalk(tmpdir_factory): + stackwalk = tmpdir_factory.mktemp("stackwalk_binary").join("stackwalk") + stackwalk.write("fake binary") + stackwalk.chmod(0o744) + return stackwalk + + +@pytest.fixture +def check_for_crashes(tmpdir, stackwalk, monkeypatch): + monkeypatch.delenv("MINIDUMP_SAVE_PATH", raising=False) + + def wrapper( + dump_directory=fspath(tmpdir), + symbols_path="symbols_path", + stackwalk_binary=fspath(stackwalk), + dump_save_path=None, + test_name=None, + quiet=True, + ): + return mozcrash.check_for_crashes( + dump_directory, + symbols_path, + stackwalk_binary, + dump_save_path, + test_name, + quiet, + ) + + return wrapper + + +@pytest.fixture +def check_for_java_exception(): + def wrapper(logcat=None, test_name=None, quiet=True): + return mozcrash.check_for_java_exception(logcat, test_name, quiet) + + return wrapper + + +def minidump_files(request, tmpdir): + files = [] + + for i in range(getattr(request, "param", 1)): + name = uuid.uuid4() + + dmp = tmpdir.join("{}.dmp".format(name)) + dmp.write("foo") + + extra = tmpdir.join("{}.extra".format(name)) + + extra.write_text( + u""" +{ + "ContentSandboxLevel":"2", + "TelemetryEnvironment":"{🍪}", + "EMCheckCompatibility":"true", + "ProductName":"Firefox", + "ContentSandboxCapabilities":"119", + "TelemetryClientId":"", + "Vendor":"Mozilla", + "InstallTime":"1000000000", + "Theme":"classic/1.0", + "ReleaseChannel":"default", + "ServerURL":"https://crash-reports.mozilla.com", + "SafeMode":"0", + "ContentSandboxCapable":"1", + "useragent_locale":"en-US", + "Version":"55.0a1", + "BuildID":"20170512114708", + "ProductID":"{ec8030f7-c20a-464f-9b0e-13a3a9e97384}", + "MozCrashReason": "MOZ_CRASH()", + "TelemetryServerURL":"", + "DOMIPCEnabled":"1", + "Add-ons":"", + "CrashTime":"1494582646", + "UptimeTS":"14.9179586", + "ContentSandboxEnabled":"1", + "ProcessType":"content", + "StartupTime":"1000000000", + "URL":"about:home" +} + + """, + encoding="utf-8", + ) + + files.append({"dmp": dmp, "extra": extra}) + + return files + + +@pytest.fixture(name="minidump_files") +def minidump_files_fixture(request, tmpdir): + return minidump_files(request, tmpdir) + + +@pytest.fixture(autouse=True) +def mock_popen(monkeypatch): + """Generate a class that can mock subprocess.Popen. + + :param stdouts: Iterable that should return an iterable for the + stdout of each process in turn. + """ + + class MockPopen(object): + def __init__(self, args, *args_rest, **kwargs): + # all_popens.append(self) + self.args = args + self.returncode = 0 + + def communicate(self): + return (u"Stackwalk command: {}".format(" ".join(self.args)), "") + + def wait(self): + return self.returncode + + monkeypatch.setattr(mozcrash.mozcrash.subprocess, "Popen", MockPopen) diff --git a/testing/mozbase/mozcrash/tests/manifest.ini b/testing/mozbase/mozcrash/tests/manifest.ini new file mode 100644 index 0000000000..5a7e75b832 --- /dev/null +++ b/testing/mozbase/mozcrash/tests/manifest.ini @@ -0,0 +1,7 @@ +[DEFAULT] +subsuite = mozbase +[test_basic.py] +[test_java_exception.py] +[test_save_path.py] +[test_stackwalk.py] +[test_symbols_path.py] diff --git a/testing/mozbase/mozcrash/tests/test_basic.py b/testing/mozbase/mozcrash/tests/test_basic.py new file mode 100644 index 0000000000..84fb2587eb --- /dev/null +++ b/testing/mozbase/mozcrash/tests/test_basic.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# coding=UTF-8 + +import mozunit +import pytest +from conftest import fspath + + +def test_no_dump_files(check_for_crashes): + """Test that check_for_crashes returns 0 if no dumps are present.""" + assert 0 == check_for_crashes() + + +@pytest.mark.parametrize("minidump_files", [3], indirect=True) +def test_dump_count(check_for_crashes, minidump_files): + """Test that check_for_crashes returns the number of crash dumps.""" + assert 3 == check_for_crashes() + + +def test_dump_directory_unicode(request, check_for_crashes, tmpdir, capsys): + """Test that check_for_crashes can handle unicode in dump_directory.""" + from conftest import minidump_files + + tmpdir = tmpdir.ensure(u"🍪", dir=1) + minidump_files = minidump_files(request, tmpdir) + + assert 1 == check_for_crashes(dump_directory=fspath(tmpdir), quiet=False) + + out, _ = capsys.readouterr() + assert fspath(minidump_files[0]["dmp"]) in out + assert u"🍪" in out + + +def test_test_name_unicode(check_for_crashes, minidump_files, capsys): + """Test that check_for_crashes can handle unicode in dump_directory.""" + assert 1 == check_for_crashes(test_name=u"🍪", quiet=False) + + out, err = capsys.readouterr() + assert u"| 🍪" in out + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/mozcrash/tests/test_java_exception.py b/testing/mozbase/mozcrash/tests/test_java_exception.py new file mode 100644 index 0000000000..f48ce2cc3e --- /dev/null +++ b/testing/mozbase/mozcrash/tests/test_java_exception.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# coding=UTF-8 + +import mozunit +import pytest + + +@pytest.fixture +def test_log(): + return [ + "01-30 20:15:41.937 E/GeckoAppShell( 1703): >>> " + 'REPORTING UNCAUGHT EXCEPTION FROM THREAD 9 ("GeckoBackgroundThread")', + "01-30 20:15:41.937 E/GeckoAppShell( 1703): java.lang.NullPointerException", + "01-30 20:15:41.937 E/GeckoAppShell( 1703):" + " at org.mozilla.gecko.GeckoApp$21.run(GeckoApp.java:1833)", + "01-30 20:15:41.937 E/GeckoAppShell( 1703):" + " at android.os.Handler.handleCallback(Handler.java:587)", + ] + + +def test_uncaught_exception(check_for_java_exception, test_log): + """Test for an exception which should be caught.""" + assert 1 == check_for_java_exception(test_log) + + +def test_truncated_exception(check_for_java_exception, test_log): + """Test for an exception which should be caught which was truncated.""" + truncated_log = list(test_log) + truncated_log[0], truncated_log[1] = truncated_log[1], truncated_log[0] + + assert 1 == check_for_java_exception(truncated_log) + + +def test_unchecked_exception(check_for_java_exception, test_log): + """Test for an exception which should not be caught.""" + passable_log = list(test_log) + passable_log[0] = ( + "01-30 20:15:41.937 E/GeckoAppShell( 1703):" + ' >>> NOT-SO-BAD EXCEPTION FROM THREAD 9 ("GeckoBackgroundThread")' + ) + + assert 0 == check_for_java_exception(passable_log) + + +def test_test_name_unicode(check_for_java_exception, test_log): + """Test that check_for_crashes can handle unicode in dump_directory.""" + assert 1 == check_for_java_exception(test_log, test_name=u"🍪", quiet=False) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/mozcrash/tests/test_save_path.py b/testing/mozbase/mozcrash/tests/test_save_path.py new file mode 100644 index 0000000000..fad83ab71b --- /dev/null +++ b/testing/mozbase/mozcrash/tests/test_save_path.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python + +import os + +import mozunit +import pytest +from conftest import fspath + + +def test_save_path_not_present(check_for_crashes, minidump_files, tmpdir): + """Test that dump_save_path works when the directory doesn't exist.""" + save_path = tmpdir.join("saved") + + assert 1 == check_for_crashes(dump_save_path=fspath(save_path)) + + assert save_path.join(minidump_files[0]["dmp"].basename).check() + assert save_path.join(minidump_files[0]["extra"].basename).check() + + +def test_save_path(check_for_crashes, minidump_files, tmpdir): + """Test that dump_save_path works.""" + save_path = tmpdir.mkdir("saved") + + assert 1 == check_for_crashes(dump_save_path=fspath(save_path)) + + assert save_path.join(minidump_files[0]["dmp"].basename).check() + assert save_path.join(minidump_files[0]["extra"].basename).check() + + +def test_save_path_isfile(check_for_crashes, minidump_files, tmpdir): + """Test that dump_save_path works when the path is a file and not a directory.""" + save_path = tmpdir.join("saved") + save_path.write("junk") + + assert 1 == check_for_crashes(dump_save_path=fspath(save_path)) + + assert save_path.join(minidump_files[0]["dmp"].basename).check() + assert save_path.join(minidump_files[0]["extra"].basename).check() + + +def test_save_path_envvar(check_for_crashes, minidump_files, tmpdir): + """Test that the MINDUMP_SAVE_PATH environment variable works.""" + save_path = tmpdir.mkdir("saved") + + os.environ["MINIDUMP_SAVE_PATH"] = fspath(save_path) + try: + assert 1 == check_for_crashes(dump_save_path=None) + finally: + del os.environ["MINIDUMP_SAVE_PATH"] + + assert save_path.join(minidump_files[0]["dmp"].basename).check() + assert save_path.join(minidump_files[0]["extra"].basename).check() + + +@pytest.mark.parametrize("minidump_files", [3], indirect=True) +def test_save_multiple(check_for_crashes, minidump_files, tmpdir): + """Test that all minidumps are saved.""" + save_path = tmpdir.mkdir("saved") + + assert 3 == check_for_crashes(dump_save_path=fspath(save_path)) + + for i in range(3): + assert save_path.join(minidump_files[i]["dmp"].basename).check() + assert save_path.join(minidump_files[i]["extra"].basename).check() + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/mozcrash/tests/test_stackwalk.py b/testing/mozbase/mozcrash/tests/test_stackwalk.py new file mode 100644 index 0000000000..7be2d82c10 --- /dev/null +++ b/testing/mozbase/mozcrash/tests/test_stackwalk.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# coding=UTF-8 + +import os + +import mozunit +from conftest import fspath + + +def test_stackwalk_not_found(check_for_crashes, minidump_files, tmpdir, capsys): + """Test that check_for_crashes can handle unicode in dump_directory.""" + stackwalk = tmpdir.join("stackwalk") + + assert 1 == check_for_crashes(stackwalk_binary=fspath(stackwalk), quiet=False) + + out, _ = capsys.readouterr() + assert "MINIDUMP_STACKWALK binary not found" in out + + +def test_stackwalk_envvar(check_for_crashes, minidump_files, stackwalk): + """Test that check_for_crashes uses the MINIDUMP_STACKWALK environment var.""" + os.environ["MINIDUMP_STACKWALK"] = fspath(stackwalk) + try: + assert 1 == check_for_crashes(stackwalk_binary=None) + finally: + del os.environ["MINIDUMP_STACKWALK"] + + +def test_stackwalk_unicode(check_for_crashes, minidump_files, tmpdir, capsys): + """Test that check_for_crashes can handle unicode in dump_directory.""" + stackwalk = tmpdir.mkdir(u"🍪").join("stackwalk") + stackwalk.write("fake binary") + stackwalk.chmod(0o744) + + assert 1 == check_for_crashes(stackwalk_binary=fspath(stackwalk), quiet=False) + + out, err = capsys.readouterr() + assert fspath(stackwalk) in out + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/mozcrash/tests/test_symbols_path.py b/testing/mozbase/mozcrash/tests/test_symbols_path.py new file mode 100644 index 0000000000..9aa281bd31 --- /dev/null +++ b/testing/mozbase/mozcrash/tests/test_symbols_path.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# coding=UTF-8 + +import zipfile + +import mozhttpd +import mozunit +from conftest import fspath +from six import BytesIO +from six.moves.urllib.parse import urlunsplit + + +def test_symbols_path_not_present(check_for_crashes, minidump_files): + """Test that no symbols path let mozcrash try to find the symbols.""" + assert 1 == check_for_crashes(symbols_path=None) + + +def test_symbols_path_unicode(check_for_crashes, minidump_files, tmpdir, capsys): + """Test that check_for_crashes can handle unicode in dump_directory.""" + symbols_path = tmpdir.mkdir(u"🍪") + + assert 1 == check_for_crashes(symbols_path=fspath(symbols_path), quiet=False) + + out, _ = capsys.readouterr() + assert fspath(symbols_path) in out + + +def test_symbols_path_url(check_for_crashes, minidump_files): + """Test that passing a URL as symbols_path correctly fetches the URL.""" + data = {"retrieved": False} + + def make_zipfile(): + zdata = BytesIO() + z = zipfile.ZipFile(zdata, "w") + z.writestr("symbols.txt", "abc/xyz") + z.close() + return zdata.getvalue() + + def get_symbols(req): + data["retrieved"] = True + + headers = {} + return (200, headers, make_zipfile()) + + httpd = mozhttpd.MozHttpd( + port=0, + urlhandlers=[{"method": "GET", "path": "/symbols", "function": get_symbols}], + ) + httpd.start() + symbol_url = urlunsplit( + ("http", "%s:%d" % httpd.httpd.server_address, "/symbols", "", "") + ) + + assert 1 == check_for_crashes(symbols_path=symbol_url) + assert data["retrieved"] + + +def test_symbols_retry(check_for_crashes, minidump_files): + """Test that passing a URL as symbols_path succeeds on retry after temporary HTTP failure.""" + data = {"retrieved": False} + get_symbols_calls = 0 + + def make_zipfile(): + zdata = BytesIO() + z = zipfile.ZipFile(zdata, "w") + z.writestr("symbols.txt", "abc/xyz") + z.close() + return zdata.getvalue() + + def get_symbols(req): + nonlocal get_symbols_calls + data["retrieved"] = True + if get_symbols_calls > 0: + ret = 200 + else: + ret = 504 + get_symbols_calls += 1 + + headers = {} + return (ret, headers, make_zipfile()) + + httpd = mozhttpd.MozHttpd( + port=0, + urlhandlers=[{"method": "GET", "path": "/symbols", "function": get_symbols}], + ) + httpd.start() + symbol_url = urlunsplit( + ("http", "%s:%d" % httpd.httpd.server_address, "/symbols", "", "") + ) + + assert 1 == check_for_crashes(symbols_path=symbol_url) + assert data["retrieved"] + assert 2 == get_symbols_calls + + +if __name__ == "__main__": + mozunit.main() |