diff options
Diffstat (limited to 'toolkit/crashreporter/tools/symbolstore.py')
| -rwxr-xr-x | toolkit/crashreporter/tools/symbolstore.py | 1096 |
1 files changed, 1096 insertions, 0 deletions
diff --git a/toolkit/crashreporter/tools/symbolstore.py b/toolkit/crashreporter/tools/symbolstore.py new file mode 100755 index 0000000000..5dd5570a84 --- /dev/null +++ b/toolkit/crashreporter/tools/symbolstore.py @@ -0,0 +1,1096 @@ +#!/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Usage: symbolstore.py <params> <dump_syms path> <symbol store path> +# <debug info files or dirs> +# Runs dump_syms on each debug info file specified on the command line, +# then places the resulting symbol file in the proper directory +# structure in the symbol store path. Accepts multiple files +# on the command line, so can be called as part of a pipe using +# find <dir> | xargs symbolstore.pl <dump_syms> <storepath> +# But really, you might just want to pass it <dir>. +# +# Parameters accepted: +# -c : Copy debug info files to the same directory structure +# as sym files. On Windows, this will also copy +# binaries into the symbol store. +# -a "<archs>" : Run dump_syms -a <arch> for each space separated +# cpu architecture in <archs> (only on OS X) +# -s <srcdir> : Use <srcdir> as the top source directory to +# generate relative filenames. + +import ctypes +import errno +import os +import platform +import re +import shutil +import subprocess +import sys +import textwrap +import time +from optparse import OptionParser +from pathlib import Path + +import buildconfig +from mozbuild.generated_sources import ( + GENERATED_SOURCE_EXTS, + get_filename_with_digest, + get_s3_region_and_bucket, +) +from mozbuild.util import memoize +from mozpack import executables +from mozpack.copier import FileRegistry +from mozpack.manifests import InstallManifest, UnreadableInstallManifest + +# Utility classes + + +class VCSFileInfo: + """A base class for version-controlled file information. Ensures that the + following attributes are generated only once (successfully): + + self.root + self.clean_root + self.revision + self.filename + + The attributes are generated by a single call to the GetRoot, + GetRevision, and GetFilename methods. Those methods are explicitly not + implemented here and must be implemented in derived classes.""" + + def __init__(self, file): + if not file: + raise ValueError + self.file = file + + def __getattr__(self, name): + """__getattr__ is only called for attributes that are not set on self, + so setting self.[attr] will prevent future calls to the GetRoot, + GetRevision, and GetFilename methods. We don't set the values on + failure on the off chance that a future call might succeed.""" + + if name == "root": + root = self.GetRoot() + if root: + self.root = root + return root + + elif name == "clean_root": + clean_root = self.GetCleanRoot() + if clean_root: + self.clean_root = clean_root + return clean_root + + elif name == "revision": + revision = self.GetRevision() + if revision: + self.revision = revision + return revision + + elif name == "filename": + filename = self.GetFilename() + if filename: + self.filename = filename + return filename + + raise AttributeError + + def GetRoot(self): + """This method should return the unmodified root for the file or 'None' + on failure.""" + raise NotImplementedError + + def GetCleanRoot(self): + """This method should return the repository root for the file or 'None' + on failure.""" + raise NotImplementedError + + def GetRevision(self): + """This method should return the revision number for the file or 'None' + on failure.""" + raise NotImplementedError + + def GetFilename(self): + """This method should return the repository-specific filename for the + file or 'None' on failure.""" + raise NotImplementedError + + +# This regex separates protocol and optional username/password from a url. +# For instance, all the following urls will be transformed into +# 'foo.com/bar': +# +# http://foo.com/bar +# svn+ssh://user@foo.com/bar +# svn+ssh://user:pass@foo.com/bar +# +rootRegex = re.compile(r"^\S+?:/+(?:[^\s/]*@)?(\S+)$") + + +def read_output(*args): + (stdout, _) = subprocess.Popen( + args=args, universal_newlines=True, stdout=subprocess.PIPE + ).communicate() + return stdout.rstrip() + + +class HGRepoInfo: + def __init__(self, path): + self.path = path + + rev = os.environ.get("MOZ_SOURCE_CHANGESET") + if not rev: + rev = read_output("hg", "-R", path, "parent", "--template={node}") + + # Look for the default hg path. If MOZ_SOURCE_REPO is set, we + # don't bother asking hg. + hg_root = os.environ.get("MOZ_SOURCE_REPO") + if hg_root: + root = hg_root + else: + root = read_output("hg", "-R", path, "showconfig", "paths.default") + if not root: + print("Failed to get HG Repo for %s" % path, file=sys.stderr) + cleanroot = None + if root: + match = rootRegex.match(root) + if match: + cleanroot = match.group(1) + if cleanroot.endswith("/"): + cleanroot = cleanroot[:-1] + if cleanroot is None: + print( + textwrap.dedent( + """\ + Could not determine repo info for %s. This is either not a clone of the web-based + repository, or you have not specified MOZ_SOURCE_REPO, or the clone is corrupt.""" + ) + % path, + sys.stderr, + ) + sys.exit(1) + self.rev = rev + self.root = root + self.cleanroot = cleanroot + + def GetFileInfo(self, file): + return HGFileInfo(file, self) + + +class HGFileInfo(VCSFileInfo): + def __init__(self, file, repo): + VCSFileInfo.__init__(self, file) + self.repo = repo + self.file = os.path.relpath(file, repo.path) + + def GetRoot(self): + return self.repo.root + + def GetCleanRoot(self): + return self.repo.cleanroot + + def GetRevision(self): + return self.repo.rev + + def GetFilename(self): + if self.revision and self.clean_root: + return "hg:%s:%s:%s" % (self.clean_root, self.file, self.revision) + return self.file + + +class GitRepoInfo: + """ + Info about a local git repository. Does not currently + support discovering info about a git clone, the info must be + provided out-of-band. + """ + + def __init__(self, path, rev, root): + self.path = path + cleanroot = None + if root: + match = rootRegex.match(root) + if match: + cleanroot = match.group(1) + if cleanroot.endswith("/"): + cleanroot = cleanroot[:-1] + if cleanroot is None: + print( + textwrap.dedent( + """\ + Could not determine repo info for %s (%s). This is either not a clone of a web-based + repository, or you have not specified MOZ_SOURCE_REPO, or the clone is corrupt.""" + ) + % (path, root), + file=sys.stderr, + ) + sys.exit(1) + self.rev = rev + self.cleanroot = cleanroot + + def GetFileInfo(self, file): + return GitFileInfo(file, self) + + +class GitFileInfo(VCSFileInfo): + def __init__(self, file, repo): + VCSFileInfo.__init__(self, file) + self.repo = repo + self.file = os.path.relpath(file, repo.path) + + def GetRoot(self): + return self.repo.path + + def GetCleanRoot(self): + return self.repo.cleanroot + + def GetRevision(self): + return self.repo.rev + + def GetFilename(self): + if self.revision and self.clean_root: + return "git:%s:%s:%s" % (self.clean_root, self.file, self.revision) + return self.file + + +# Utility functions + + +# A cache of files for which VCS info has already been determined. Used to +# prevent extra filesystem activity or process launching. +vcsFileInfoCache = {} + +if platform.system() == "Windows": + + def realpath(path): + """ + Normalize a path using `GetFinalPathNameByHandleW` to get the + path with all components in the case they exist in on-disk, so + that making links to a case-sensitive server (hg.mozilla.org) works. + + This function also resolves any symlinks in the path. + """ + # Return the original path if something fails, which can happen for paths that + # don't exist on this system (like paths from the CRT). + result = path + + ctypes.windll.kernel32.SetErrorMode(ctypes.c_uint(1)) + handle = ctypes.windll.kernel32.CreateFileW( + path, + # GENERIC_READ + 0x80000000, + # FILE_SHARE_READ + 1, + None, + # OPEN_EXISTING + 3, + # FILE_FLAG_BACKUP_SEMANTICS + # This is necessary to open + # directory handles. + 0x02000000, + None, + ) + if handle != -1: + size = ctypes.windll.kernel32.GetFinalPathNameByHandleW(handle, None, 0, 0) + buf = ctypes.create_unicode_buffer(size) + if ( + ctypes.windll.kernel32.GetFinalPathNameByHandleW(handle, buf, size, 0) + > 0 + ): + # The return value of GetFinalPathNameByHandleW uses the + # '\\?\' prefix. + result = buf.value[4:] + ctypes.windll.kernel32.CloseHandle(handle) + return result + + +else: + # Just use the os.path version otherwise. + realpath = os.path.realpath + + +def IsInDir(file, dir): + try: + Path(file).relative_to(dir) + return True + except ValueError: + return False + + +def GetVCSFilenameFromSrcdir(file, srcdir): + if srcdir not in Dumper.srcdirRepoInfo: + # Not in cache, so find it adnd cache it + if os.path.isdir(os.path.join(srcdir, ".hg")): + Dumper.srcdirRepoInfo[srcdir] = HGRepoInfo(srcdir) + else: + # Unknown VCS or file is not in a repo. + return None + return Dumper.srcdirRepoInfo[srcdir].GetFileInfo(file) + + +def GetVCSFilename(file, srcdirs): + """Given a full path to a file, and the top source directory, + look for version control information about this file, and return + a tuple containing + 1) a specially formatted filename that contains the VCS type, + VCS location, relative filename, and revision number, formatted like: + vcs:vcs location:filename:revision + For example: + cvs:cvs.mozilla.org/cvsroot:mozilla/browser/app/nsBrowserApp.cpp:1.36 + 2) the unmodified root information if it exists""" + (path, filename) = os.path.split(file) + if path == "" or filename == "": + return (file, None) + + fileInfo = None + root = "" + if file in vcsFileInfoCache: + # Already cached this info, use it. + fileInfo = vcsFileInfoCache[file] + else: + for srcdir in srcdirs: + if not IsInDir(file, srcdir): + continue + fileInfo = GetVCSFilenameFromSrcdir(file, srcdir) + if fileInfo: + vcsFileInfoCache[file] = fileInfo + break + + if fileInfo: + file = fileInfo.filename + root = fileInfo.root + + # we want forward slashes on win32 paths + return (file.replace("\\", "/"), root) + + +def validate_install_manifests(install_manifest_args): + args = [] + for arg in install_manifest_args: + bits = arg.split(",") + if len(bits) != 2: + raise ValueError( + "Invalid format for --install-manifest: " "specify manifest,target_dir" + ) + manifest_file, destination = [os.path.abspath(b) for b in bits] + if not os.path.isfile(manifest_file): + raise IOError(errno.ENOENT, "Manifest file not found", manifest_file) + if not os.path.isdir(destination): + raise IOError(errno.ENOENT, "Install directory not found", destination) + try: + manifest = InstallManifest(manifest_file) + except UnreadableInstallManifest: + raise IOError(errno.EINVAL, "Error parsing manifest file", manifest_file) + args.append((manifest, destination)) + return args + + +def make_file_mapping(install_manifests): + file_mapping = {} + for manifest, destination in install_manifests: + destination = os.path.abspath(destination) + reg = FileRegistry() + manifest.populate_registry(reg) + for dst, src in reg: + if hasattr(src, "path"): + # Any paths that get compared to source file names need to go through realpath. + abs_dest = realpath(os.path.join(destination, dst)) + file_mapping[abs_dest] = realpath(src.path) + return file_mapping + + +@memoize +def get_generated_file_s3_path(filename, rel_path, bucket): + """Given a filename, return a path formatted similarly to + GetVCSFilename but representing a file available in an s3 bucket.""" + with open(filename, "rb") as f: + path = get_filename_with_digest(rel_path, f.read()) + return "s3:{bucket}:{path}:".format(bucket=bucket, path=path) + + +def GetPlatformSpecificDumper(**kwargs): + """This function simply returns a instance of a subclass of Dumper + that is appropriate for the current platform.""" + return {"WINNT": Dumper_Win32, "Linux": Dumper_Linux, "Darwin": Dumper_Mac}[ + buildconfig.substs["OS_ARCH"] + ](**kwargs) + + +def SourceIndex(fileStream, outputPath, vcs_root, s3_bucket): + """Takes a list of files, writes info to a data block in a .stream file""" + # Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing + # Create the srcsrv data block that indexes the pdb file + result = True + pdbStreamFile = open(outputPath, "w") + pdbStreamFile.write( + "SRCSRV: ini ------------------------------------------------\r\n" + + "VERSION=2\r\n" + + "INDEXVERSION=2\r\n" + + "VERCTRL=http\r\n" + + "SRCSRV: variables ------------------------------------------\r\n" + + "SRCSRVVERCTRL=http\r\n" + + "RUST_GITHUB_TARGET=https://github.com/rust-lang/rust/raw/%var4%/%var3%\r\n" + ) + pdbStreamFile.write("HGSERVER=" + vcs_root + "\r\n") + pdbStreamFile.write("HG_TARGET=%hgserver%/raw-file/%var4%/%var3%\r\n") + + if s3_bucket: + pdbStreamFile.write("S3_BUCKET=" + s3_bucket + "\r\n") + pdbStreamFile.write("S3_TARGET=https://%s3_bucket%.s3.amazonaws.com/%var3%\r\n") + + # Allow each entry to choose its template via "var2". + # Possible values for var2 are: HG_TARGET / S3_TARGET / RUST_GITHUB_TARGET + pdbStreamFile.write("SRCSRVTRG=%fnvar%(%var2%)\r\n") + + pdbStreamFile.write( + "SRCSRV: source files ---------------------------------------\r\n" + ) + pdbStreamFile.write(fileStream) + pdbStreamFile.write( + "SRCSRV: end ------------------------------------------------\r\n\n" + ) + pdbStreamFile.close() + return result + + +class Dumper: + """This class can dump symbols from a file with debug info, and + store the output in a directory structure that is valid for use as + a Breakpad symbol server. Requires a path to a dump_syms binary-- + |dump_syms| and a directory to store symbols in--|symbol_path|. + Optionally takes a list of processor architectures to process from + each debug file--|archs|, the full path to the top source + directory--|srcdir|, for generating relative source file names, + and an option to copy debug info files alongside the dumped + symbol files--|copy_debug|, mostly useful for creating a + Microsoft Symbol Server from the resulting output. + + You don't want to use this directly if you intend to process files. + Instead, call GetPlatformSpecificDumper to get an instance of a + subclass.""" + + srcdirRepoInfo = {} + + def __init__( + self, + dump_syms, + symbol_path, + archs=None, + srcdirs=[], + copy_debug=False, + vcsinfo=False, + srcsrv=False, + s3_bucket=None, + file_mapping=None, + ): + # popen likes absolute paths, at least on windows + self.dump_syms = os.path.abspath(dump_syms) + self.symbol_path = symbol_path + if archs is None: + # makes the loop logic simpler + self.archs = [""] + else: + self.archs = ["-a %s" % a for a in archs.split()] + # Any paths that get compared to source file names need to go through realpath. + self.srcdirs = [realpath(s) for s in srcdirs] + self.copy_debug = copy_debug + self.vcsinfo = vcsinfo + self.srcsrv = srcsrv + self.s3_bucket = s3_bucket + self.file_mapping = file_mapping or {} + # Add a static mapping for Rust sources. Since Rust 1.30 official Rust builds map + # source paths to start with "/rust/<sha>/". + rust_sha = buildconfig.substs["RUSTC_COMMIT"] + rust_srcdir = "/rustc/" + rust_sha + self.srcdirs.append(rust_srcdir) + Dumper.srcdirRepoInfo[rust_srcdir] = GitRepoInfo( + rust_srcdir, rust_sha, "https://github.com/rust-lang/rust/" + ) + + # subclasses override this + def ShouldProcess(self, file): + return True + + # This is a no-op except on Win32 + def SourceServerIndexing( + self, debug_file, guid, sourceFileStream, vcs_root, s3_bucket + ): + return "" + + # subclasses override this if they want to support this + def CopyExeAndDebugInfo(self, file, debug_file, guid, code_file, code_id): + """This function will copy a library or executable and the file holding the + debug information to |symbol_path|""" + pass + + def Process(self, file_to_process, count_ctors=False): + """Process the given file.""" + if self.ShouldProcess(os.path.abspath(file_to_process)): + self.ProcessFile(file_to_process, count_ctors=count_ctors) + + def ProcessFile(self, file, dsymbundle=None, count_ctors=False): + """Dump symbols from these files into a symbol file, stored + in the proper directory structure in |symbol_path|; processing is performed + asynchronously, and Finish must be called to wait for it complete and cleanup. + All files after the first are fallbacks in case the first file does not process + successfully; if it does, no other files will be touched.""" + print("Beginning work for file: %s" % file, file=sys.stderr) + + # tries to get the vcs root from the .mozconfig first - if it's not set + # the tinderbox vcs path will be assigned further down + vcs_root = os.environ.get("MOZ_SOURCE_REPO") + for arch_num, arch in enumerate(self.archs): + self.ProcessFileWork( + file, arch_num, arch, vcs_root, dsymbundle, count_ctors=count_ctors + ) + + def dump_syms_cmdline(self, file, arch, dsymbundle=None): + """ + Get the commandline used to invoke dump_syms. + """ + # The Mac dumper overrides this. + return [self.dump_syms, "--inlines", file] + + def ProcessFileWork( + self, file, arch_num, arch, vcs_root, dsymbundle=None, count_ctors=False + ): + ctors = 0 + t_start = time.time() + print("Processing file: %s" % file, file=sys.stderr) + + sourceFileStream = "" + code_id, code_file = None, None + try: + cmd = self.dump_syms_cmdline(file, arch, dsymbundle=dsymbundle) + print(" ".join(cmd), file=sys.stderr) + proc = subprocess.Popen( + cmd, + universal_newlines=True, + stdout=subprocess.PIPE, + ) + try: + module_line = next(proc.stdout) + except StopIteration: + module_line = "" + if module_line.startswith("MODULE"): + # MODULE os cpu guid debug_file + (guid, debug_file) = (module_line.split())[3:5] + # strip off .pdb extensions, and append .sym + sym_file = re.sub("\.pdb$", "", debug_file) + ".sym" + # we do want forward slashes here + rel_path = os.path.join(debug_file, guid, sym_file).replace("\\", "/") + full_path = os.path.normpath(os.path.join(self.symbol_path, rel_path)) + try: + os.makedirs(os.path.dirname(full_path)) + except OSError: # already exists + pass + f = open(full_path, "w") + f.write(module_line) + # now process the rest of the output + for line in proc.stdout: + if line.startswith("FILE"): + # FILE index filename + (x, index, filename) = line.rstrip().split(None, 2) + # We want original file paths for the source server. + sourcepath = filename + filename = realpath(filename) + if filename in self.file_mapping: + filename = self.file_mapping[filename] + if self.vcsinfo: + try: + gen_path = Path(filename) + rel_gen_path = gen_path.relative_to( + buildconfig.topobjdir + ) + except ValueError: + gen_path = None + if ( + gen_path + and gen_path.exists() + and gen_path.suffix in GENERATED_SOURCE_EXTS + and self.s3_bucket + ): + filename = get_generated_file_s3_path( + filename, str(rel_gen_path), self.s3_bucket + ) + rootname = "" + else: + (filename, rootname) = GetVCSFilename( + filename, self.srcdirs + ) + # sets vcs_root in case the loop through files were to end + # on an empty rootname + if vcs_root is None: + if rootname: + vcs_root = rootname + # Emit an entry for the file mapping for the srcsrv stream + if filename.startswith("hg:"): + (vcs, repo, source_file, revision) = filename.split(":", 3) + sourceFileStream += sourcepath + "*HG_TARGET*" + source_file + sourceFileStream += "*" + revision + "\r\n" + elif filename.startswith("s3:"): + (vcs, bucket, source_file, nothing) = filename.split(":", 3) + sourceFileStream += sourcepath + "*S3_TARGET*" + sourceFileStream += source_file + "\r\n" + elif filename.startswith("git:github.com/rust-lang/rust:"): + (vcs, repo, source_file, revision) = filename.split(":", 3) + sourceFileStream += sourcepath + "*RUST_GITHUB_TARGET*" + sourceFileStream += source_file + "*" + revision + "\r\n" + f.write("FILE %s %s\n" % (index, filename)) + elif line.startswith("INFO CODE_ID "): + # INFO CODE_ID code_id code_file + # This gives some info we can use to + # store binaries in the symbol store. + bits = line.rstrip().split(None, 3) + if len(bits) == 4: + code_id, code_file = bits[2:] + f.write(line) + else: + if count_ctors and line.startswith("FUNC "): + # Static initializers, as created by clang and gcc + # have symbols that start with "_GLOBAL_sub" + if "_GLOBAL__sub_" in line: + ctors += 1 + # MSVC creates `dynamic initializer for '...'` + # symbols. + elif "`dynamic initializer for '" in line: + ctors += 1 + + # pass through all other lines unchanged + f.write(line) + f.close() + retcode = proc.wait() + if retcode != 0: + raise RuntimeError( + "dump_syms failed with error code %d while processing %s\n" + % (retcode, file) + ) + # we output relative paths so callers can get a list of what + # was generated + print(rel_path) + if self.srcsrv and vcs_root: + # add source server indexing to the pdb file + self.SourceServerIndexing( + debug_file, guid, sourceFileStream, vcs_root, self.s3_bucket + ) + # only copy debug the first time if we have multiple architectures + if self.copy_debug and arch_num == 0: + self.CopyExeAndDebugInfo(file, debug_file, guid, code_file, code_id) + else: + # For some reason, we didn't see the MODULE line as the first + # line of output, this is strictly required so fail irrespective + # of the process' return code. + retcode = proc.wait() + message = [ + "dump_syms failed to produce the expected output", + "file: %s" % file, + "return code: %d" % retcode, + "first line of output: %s" % module_line, + ] + raise RuntimeError("\n----------\n".join(message)) + except Exception as e: + print("Unexpected error: %s" % str(e), file=sys.stderr) + raise + + if dsymbundle: + shutil.rmtree(dsymbundle) + + if count_ctors: + import json + + perfherder_data = { + "framework": {"name": "build_metrics"}, + "suites": [ + { + "name": "compiler_metrics", + "subtests": [ + { + "name": "num_static_constructors", + "value": ctors, + "alertChangeType": "absolute", + "alertThreshold": 3, + } + ], + } + ], + } + perfherder_extra_options = os.environ.get("PERFHERDER_EXTRA_OPTIONS", "") + for opt in perfherder_extra_options.split(): + for suite in perfherder_data["suites"]: + if opt not in suite.get("extraOptions", []): + suite.setdefault("extraOptions", []).append(opt) + + if "asan" not in perfherder_extra_options.lower(): + print( + "PERFHERDER_DATA: %s" % json.dumps(perfherder_data), file=sys.stderr + ) + + elapsed = time.time() - t_start + print("Finished processing %s in %.2fs" % (file, elapsed), file=sys.stderr) + + +# Platform-specific subclasses. For the most part, these just have +# logic to determine what files to extract symbols from. + + +def locate_pdb(path): + """Given a path to a binary, attempt to locate the matching pdb file with simple heuristics: + * Look for a pdb file with the same base name next to the binary + * Look for a pdb file with the same base name in the cwd + + Returns the path to the pdb file if it exists, or None if it could not be located. + """ + path, ext = os.path.splitext(path) + pdb = path + ".pdb" + if os.path.isfile(pdb): + return pdb + # If there's no pdb next to the file, see if there's a pdb with the same root name + # in the cwd. We build some binaries directly into dist/bin, but put the pdb files + # in the relative objdir, which is the cwd when running this script. + base = os.path.basename(pdb) + pdb = os.path.join(os.getcwd(), base) + if os.path.isfile(pdb): + return pdb + return None + + +class Dumper_Win32(Dumper): + fixedFilenameCaseCache = {} + + def ShouldProcess(self, file): + """This function will allow processing of exe or dll files that have pdb + files with the same base name next to them.""" + if file.endswith(".exe") or file.endswith(".dll"): + if locate_pdb(file) is not None: + return True + return False + + def CopyExeAndDebugInfo(self, file, debug_file, guid, code_file, code_id): + """This function will copy the executable or dll and pdb files to |symbol_path|""" + pdb_file = locate_pdb(file) + + rel_path = os.path.join(debug_file, guid, debug_file).replace("\\", "/") + full_path = os.path.normpath(os.path.join(self.symbol_path, rel_path)) + shutil.copyfile(pdb_file, full_path) + print(rel_path) + + # Copy the binary file as well + if code_file and code_id: + full_code_path = os.path.join(os.path.dirname(file), code_file) + if os.path.exists(full_code_path): + rel_path = os.path.join(code_file, code_id, code_file).replace( + "\\", "/" + ) + full_path = os.path.normpath(os.path.join(self.symbol_path, rel_path)) + try: + os.makedirs(os.path.dirname(full_path)) + except OSError as e: + if e.errno != errno.EEXIST: + raise + shutil.copyfile(full_code_path, full_path) + print(rel_path) + + def SourceServerIndexing( + self, debug_file, guid, sourceFileStream, vcs_root, s3_bucket + ): + # Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing + streamFilename = debug_file + ".stream" + stream_output_path = os.path.abspath(streamFilename) + # Call SourceIndex to create the .stream file + result = SourceIndex(sourceFileStream, stream_output_path, vcs_root, s3_bucket) + if self.copy_debug: + pdbstr = buildconfig.substs["PDBSTR"] + wine = buildconfig.substs.get("WINE") + if wine: + cmd = [wine, pdbstr] + else: + cmd = [pdbstr] + subprocess.call( + cmd + + [ + "-w", + "-p:" + os.path.basename(debug_file), + "-i:" + os.path.basename(streamFilename), + "-s:srcsrv", + ], + cwd=os.path.dirname(stream_output_path), + ) + # clean up all the .stream files when done + os.remove(stream_output_path) + return result + + +class Dumper_Linux(Dumper): + objcopy = os.environ["OBJCOPY"] if "OBJCOPY" in os.environ else "objcopy" + + def ShouldProcess(self, file): + """This function will allow processing of files that are + executable, or end with the .so extension, and additionally + file(1) reports as being ELF files. It expects to find the file + command in PATH.""" + if file.endswith(".so") or os.access(file, os.X_OK): + return executables.get_type(file) == executables.ELF + return False + + def CopyExeAndDebugInfo(self, file, debug_file, guid, code_file, code_id): + # We want to strip out the debug info, and add a + # .gnu_debuglink section to the object, so the debugger can + # actually load our debug info later. + # In some odd cases, the object might already have an irrelevant + # .gnu_debuglink section, and objcopy doesn't want to add one in + # such cases, so we make it remove it any existing one first. + file_dbg = file + ".dbg" + if ( + subprocess.call([self.objcopy, "--only-keep-debug", file, file_dbg]) == 0 + and subprocess.call( + [ + self.objcopy, + "--remove-section", + ".gnu_debuglink", + "--add-gnu-debuglink=%s" % file_dbg, + file, + ] + ) + == 0 + ): + rel_path = os.path.join(debug_file, guid, debug_file + ".dbg") + full_path = os.path.normpath(os.path.join(self.symbol_path, rel_path)) + shutil.move(file_dbg, full_path) + print(rel_path) + else: + if os.path.isfile(file_dbg): + os.unlink(file_dbg) + + +class Dumper_Solaris(Dumper): + def ShouldProcess(self, file): + """This function will allow processing of files that are + executable, or end with the .so extension, and additionally + file(1) reports as being ELF files. It expects to find the file + command in PATH.""" + if file.endswith(".so") or os.access(file, os.X_OK): + return executables.get_type(file) == executables.ELF + return False + + +class Dumper_Mac(Dumper): + def ShouldProcess(self, file): + """This function will allow processing of files that are + executable, or end with the .dylib extension, and additionally + file(1) reports as being Mach-O files. It expects to find the file + command in PATH.""" + if file.endswith(".dylib") or os.access(file, os.X_OK): + return executables.get_type(file) == executables.MACHO + return False + + def ProcessFile(self, file, count_ctors=False): + print("Starting Mac pre-processing on file: %s" % file, file=sys.stderr) + dsymbundle = self.GenerateDSYM(file) + if dsymbundle: + # kick off new jobs per-arch with our new list of files + Dumper.ProcessFile( + self, file, dsymbundle=dsymbundle, count_ctors=count_ctors + ) + + def dump_syms_cmdline(self, file, arch, dsymbundle=None): + """ + Get the commandline used to invoke dump_syms. + """ + # dump_syms wants the path to the original binary and the .dSYM + # in order to dump all the symbols. + if dsymbundle: + # This is the .dSYM bundle. + return ( + [self.dump_syms] + + arch.split() + + ["--inlines", "-j", "2", dsymbundle, file] + ) + return Dumper.dump_syms_cmdline(self, file, arch) + + def GenerateDSYM(self, file): + """dump_syms on Mac needs to be run on a dSYM bundle produced + by dsymutil(1), so run dsymutil here and pass the bundle name + down to the superclass method instead.""" + t_start = time.time() + print("Running Mac pre-processing on file: %s" % (file,), file=sys.stderr) + + dsymbundle = file + ".dSYM" + if os.path.exists(dsymbundle): + shutil.rmtree(dsymbundle) + dsymutil = buildconfig.substs["DSYMUTIL"] + # dsymutil takes --arch=foo instead of -a foo like everything else + cmd = ( + [dsymutil] + [a.replace("-a ", "--arch=") for a in self.archs if a] + [file] + ) + print(" ".join(cmd), file=sys.stderr) + + dsymutil_proc = subprocess.Popen( + cmd, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + dsymout, dsymerr = dsymutil_proc.communicate() + if dsymutil_proc.returncode != 0: + raise RuntimeError("Error running dsymutil: %s" % dsymerr) + + # Regular dsymutil won't produce a .dSYM for files without symbols. + if not os.path.exists(dsymbundle): + print("No symbols found in file: %s" % (file,), file=sys.stderr) + return False + + # llvm-dsymutil will produce a .dSYM for files without symbols or + # debug information, but only sometimes will it warn you about this. + # We don't want to run dump_syms on such bundles, because asserts + # will fire in debug mode and who knows what will happen in release. + # + # So we check for the error message and bail if it appears. If it + # doesn't, we carefully check the bundled DWARF to see if dump_syms + # will be OK with it. + if "warning: no debug symbols in" in dsymerr: + print(dsymerr, file=sys.stderr) + return False + + contents_dir = os.path.join(dsymbundle, "Contents", "Resources", "DWARF") + if not os.path.exists(contents_dir): + print( + "No DWARF information in .dSYM bundle %s" % (dsymbundle,), + file=sys.stderr, + ) + return False + + files = os.listdir(contents_dir) + if len(files) != 1: + print("Unexpected files in .dSYM bundle %s" % (files,), file=sys.stderr) + return False + + otool_out = subprocess.check_output( + [buildconfig.substs["OTOOL"], "-l", os.path.join(contents_dir, files[0])], + universal_newlines=True, + ) + if "sectname __debug_info" not in otool_out: + print("No symbols in .dSYM bundle %s" % (dsymbundle,), file=sys.stderr) + return False + + elapsed = time.time() - t_start + print("Finished processing %s in %.2fs" % (file, elapsed), file=sys.stderr) + return dsymbundle + + def CopyExeAndDebugInfo(self, file, debug_file, guid, code_file, code_id): + """ProcessFile has already produced a dSYM bundle, so we should just + copy that to the destination directory. However, we'll package it + into a .tar because it's a bundle, so it's a directory. |file| here is + the original filename.""" + dsymbundle = file + ".dSYM" + rel_path = os.path.join(debug_file, guid, os.path.basename(dsymbundle) + ".tar") + full_path = os.path.abspath(os.path.join(self.symbol_path, rel_path)) + success = subprocess.call( + ["tar", "cf", full_path, os.path.basename(dsymbundle)], + cwd=os.path.dirname(dsymbundle), + stdout=open(os.devnull, "w"), + stderr=subprocess.STDOUT, + ) + if success == 0 and os.path.exists(full_path): + print(rel_path) + + +# Entry point if called as a standalone program + + +def main(): + parser = OptionParser( + usage="usage: %prog [options] <dump_syms binary> <symbol store path> <debug info files>" + ) + parser.add_option( + "-c", + "--copy", + action="store_true", + dest="copy_debug", + default=False, + help="Copy debug info files into the same directory structure as symbol files", + ) + parser.add_option( + "-a", + "--archs", + action="store", + dest="archs", + help="Run dump_syms -a <arch> for each space separated" + + "cpu architecture in ARCHS (only on OS X)", + ) + parser.add_option( + "-s", + "--srcdir", + action="append", + dest="srcdir", + default=[], + help="Use SRCDIR to determine relative paths to source files", + ) + parser.add_option( + "-v", + "--vcs-info", + action="store_true", + dest="vcsinfo", + help="Try to retrieve VCS info for each FILE listed in the output", + ) + parser.add_option( + "-i", + "--source-index", + action="store_true", + dest="srcsrv", + default=False, + help="Add source index information to debug files, making them suitable" + + " for use in a source server.", + ) + parser.add_option( + "--install-manifest", + action="append", + dest="install_manifests", + default=[], + help="""Use this install manifest to map filenames back +to canonical locations in the source repository. Specify +<install manifest filename>,<install destination> as a comma-separated pair.""", + ) + parser.add_option( + "--count-ctors", + action="store_true", + dest="count_ctors", + default=False, + help="Count static initializers", + ) + (options, args) = parser.parse_args() + + # check to see if the pdbstr.exe exists + if options.srcsrv: + if "PDBSTR" not in buildconfig.substs: + print("pdbstr was not found by configure.\n", file=sys.stderr) + sys.exit(1) + + if len(args) < 3: + parser.error("not enough arguments") + exit(1) + + try: + manifests = validate_install_manifests(options.install_manifests) + except (IOError, ValueError) as e: + parser.error(str(e)) + exit(1) + file_mapping = make_file_mapping(manifests) + _, bucket = get_s3_region_and_bucket() + dumper = GetPlatformSpecificDumper( + dump_syms=args[0], + symbol_path=args[1], + copy_debug=options.copy_debug, + archs=options.archs, + srcdirs=options.srcdir, + vcsinfo=options.vcsinfo, + srcsrv=options.srcsrv, + s3_bucket=bucket, + file_mapping=file_mapping, + ) + + dumper.Process(args[2], options.count_ctors) + + +# run main if run directly +if __name__ == "__main__": + main() |