summaryrefslogtreecommitdiffstats
path: root/toolkit/crashreporter/tools/symbolstore.py
diff options
context:
space:
mode:
Diffstat (limited to 'toolkit/crashreporter/tools/symbolstore.py')
-rwxr-xr-xtoolkit/crashreporter/tools/symbolstore.py1096
1 files changed, 1096 insertions, 0 deletions
diff --git a/toolkit/crashreporter/tools/symbolstore.py b/toolkit/crashreporter/tools/symbolstore.py
new file mode 100755
index 0000000000..5dd5570a84
--- /dev/null
+++ b/toolkit/crashreporter/tools/symbolstore.py
@@ -0,0 +1,1096 @@
+#!/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Usage: symbolstore.py <params> <dump_syms path> <symbol store path>
+# <debug info files or dirs>
+# Runs dump_syms on each debug info file specified on the command line,
+# then places the resulting symbol file in the proper directory
+# structure in the symbol store path. Accepts multiple files
+# on the command line, so can be called as part of a pipe using
+# find <dir> | xargs symbolstore.pl <dump_syms> <storepath>
+# But really, you might just want to pass it <dir>.
+#
+# Parameters accepted:
+# -c : Copy debug info files to the same directory structure
+# as sym files. On Windows, this will also copy
+# binaries into the symbol store.
+# -a "<archs>" : Run dump_syms -a <arch> for each space separated
+# cpu architecture in <archs> (only on OS X)
+# -s <srcdir> : Use <srcdir> as the top source directory to
+# generate relative filenames.
+
+import ctypes
+import errno
+import os
+import platform
+import re
+import shutil
+import subprocess
+import sys
+import textwrap
+import time
+from optparse import OptionParser
+from pathlib import Path
+
+import buildconfig
+from mozbuild.generated_sources import (
+ GENERATED_SOURCE_EXTS,
+ get_filename_with_digest,
+ get_s3_region_and_bucket,
+)
+from mozbuild.util import memoize
+from mozpack import executables
+from mozpack.copier import FileRegistry
+from mozpack.manifests import InstallManifest, UnreadableInstallManifest
+
+# Utility classes
+
+
+class VCSFileInfo:
+ """A base class for version-controlled file information. Ensures that the
+ following attributes are generated only once (successfully):
+
+ self.root
+ self.clean_root
+ self.revision
+ self.filename
+
+ The attributes are generated by a single call to the GetRoot,
+ GetRevision, and GetFilename methods. Those methods are explicitly not
+ implemented here and must be implemented in derived classes."""
+
+ def __init__(self, file):
+ if not file:
+ raise ValueError
+ self.file = file
+
+ def __getattr__(self, name):
+ """__getattr__ is only called for attributes that are not set on self,
+ so setting self.[attr] will prevent future calls to the GetRoot,
+ GetRevision, and GetFilename methods. We don't set the values on
+ failure on the off chance that a future call might succeed."""
+
+ if name == "root":
+ root = self.GetRoot()
+ if root:
+ self.root = root
+ return root
+
+ elif name == "clean_root":
+ clean_root = self.GetCleanRoot()
+ if clean_root:
+ self.clean_root = clean_root
+ return clean_root
+
+ elif name == "revision":
+ revision = self.GetRevision()
+ if revision:
+ self.revision = revision
+ return revision
+
+ elif name == "filename":
+ filename = self.GetFilename()
+ if filename:
+ self.filename = filename
+ return filename
+
+ raise AttributeError
+
+ def GetRoot(self):
+ """This method should return the unmodified root for the file or 'None'
+ on failure."""
+ raise NotImplementedError
+
+ def GetCleanRoot(self):
+ """This method should return the repository root for the file or 'None'
+ on failure."""
+ raise NotImplementedError
+
+ def GetRevision(self):
+ """This method should return the revision number for the file or 'None'
+ on failure."""
+ raise NotImplementedError
+
+ def GetFilename(self):
+ """This method should return the repository-specific filename for the
+ file or 'None' on failure."""
+ raise NotImplementedError
+
+
+# This regex separates protocol and optional username/password from a url.
+# For instance, all the following urls will be transformed into
+# 'foo.com/bar':
+#
+# http://foo.com/bar
+# svn+ssh://user@foo.com/bar
+# svn+ssh://user:pass@foo.com/bar
+#
+rootRegex = re.compile(r"^\S+?:/+(?:[^\s/]*@)?(\S+)$")
+
+
+def read_output(*args):
+ (stdout, _) = subprocess.Popen(
+ args=args, universal_newlines=True, stdout=subprocess.PIPE
+ ).communicate()
+ return stdout.rstrip()
+
+
+class HGRepoInfo:
+ def __init__(self, path):
+ self.path = path
+
+ rev = os.environ.get("MOZ_SOURCE_CHANGESET")
+ if not rev:
+ rev = read_output("hg", "-R", path, "parent", "--template={node}")
+
+ # Look for the default hg path. If MOZ_SOURCE_REPO is set, we
+ # don't bother asking hg.
+ hg_root = os.environ.get("MOZ_SOURCE_REPO")
+ if hg_root:
+ root = hg_root
+ else:
+ root = read_output("hg", "-R", path, "showconfig", "paths.default")
+ if not root:
+ print("Failed to get HG Repo for %s" % path, file=sys.stderr)
+ cleanroot = None
+ if root:
+ match = rootRegex.match(root)
+ if match:
+ cleanroot = match.group(1)
+ if cleanroot.endswith("/"):
+ cleanroot = cleanroot[:-1]
+ if cleanroot is None:
+ print(
+ textwrap.dedent(
+ """\
+ Could not determine repo info for %s. This is either not a clone of the web-based
+ repository, or you have not specified MOZ_SOURCE_REPO, or the clone is corrupt."""
+ )
+ % path,
+ sys.stderr,
+ )
+ sys.exit(1)
+ self.rev = rev
+ self.root = root
+ self.cleanroot = cleanroot
+
+ def GetFileInfo(self, file):
+ return HGFileInfo(file, self)
+
+
+class HGFileInfo(VCSFileInfo):
+ def __init__(self, file, repo):
+ VCSFileInfo.__init__(self, file)
+ self.repo = repo
+ self.file = os.path.relpath(file, repo.path)
+
+ def GetRoot(self):
+ return self.repo.root
+
+ def GetCleanRoot(self):
+ return self.repo.cleanroot
+
+ def GetRevision(self):
+ return self.repo.rev
+
+ def GetFilename(self):
+ if self.revision and self.clean_root:
+ return "hg:%s:%s:%s" % (self.clean_root, self.file, self.revision)
+ return self.file
+
+
+class GitRepoInfo:
+ """
+ Info about a local git repository. Does not currently
+ support discovering info about a git clone, the info must be
+ provided out-of-band.
+ """
+
+ def __init__(self, path, rev, root):
+ self.path = path
+ cleanroot = None
+ if root:
+ match = rootRegex.match(root)
+ if match:
+ cleanroot = match.group(1)
+ if cleanroot.endswith("/"):
+ cleanroot = cleanroot[:-1]
+ if cleanroot is None:
+ print(
+ textwrap.dedent(
+ """\
+ Could not determine repo info for %s (%s). This is either not a clone of a web-based
+ repository, or you have not specified MOZ_SOURCE_REPO, or the clone is corrupt."""
+ )
+ % (path, root),
+ file=sys.stderr,
+ )
+ sys.exit(1)
+ self.rev = rev
+ self.cleanroot = cleanroot
+
+ def GetFileInfo(self, file):
+ return GitFileInfo(file, self)
+
+
+class GitFileInfo(VCSFileInfo):
+ def __init__(self, file, repo):
+ VCSFileInfo.__init__(self, file)
+ self.repo = repo
+ self.file = os.path.relpath(file, repo.path)
+
+ def GetRoot(self):
+ return self.repo.path
+
+ def GetCleanRoot(self):
+ return self.repo.cleanroot
+
+ def GetRevision(self):
+ return self.repo.rev
+
+ def GetFilename(self):
+ if self.revision and self.clean_root:
+ return "git:%s:%s:%s" % (self.clean_root, self.file, self.revision)
+ return self.file
+
+
+# Utility functions
+
+
+# A cache of files for which VCS info has already been determined. Used to
+# prevent extra filesystem activity or process launching.
+vcsFileInfoCache = {}
+
+if platform.system() == "Windows":
+
+ def realpath(path):
+ """
+ Normalize a path using `GetFinalPathNameByHandleW` to get the
+ path with all components in the case they exist in on-disk, so
+ that making links to a case-sensitive server (hg.mozilla.org) works.
+
+ This function also resolves any symlinks in the path.
+ """
+ # Return the original path if something fails, which can happen for paths that
+ # don't exist on this system (like paths from the CRT).
+ result = path
+
+ ctypes.windll.kernel32.SetErrorMode(ctypes.c_uint(1))
+ handle = ctypes.windll.kernel32.CreateFileW(
+ path,
+ # GENERIC_READ
+ 0x80000000,
+ # FILE_SHARE_READ
+ 1,
+ None,
+ # OPEN_EXISTING
+ 3,
+ # FILE_FLAG_BACKUP_SEMANTICS
+ # This is necessary to open
+ # directory handles.
+ 0x02000000,
+ None,
+ )
+ if handle != -1:
+ size = ctypes.windll.kernel32.GetFinalPathNameByHandleW(handle, None, 0, 0)
+ buf = ctypes.create_unicode_buffer(size)
+ if (
+ ctypes.windll.kernel32.GetFinalPathNameByHandleW(handle, buf, size, 0)
+ > 0
+ ):
+ # The return value of GetFinalPathNameByHandleW uses the
+ # '\\?\' prefix.
+ result = buf.value[4:]
+ ctypes.windll.kernel32.CloseHandle(handle)
+ return result
+
+
+else:
+ # Just use the os.path version otherwise.
+ realpath = os.path.realpath
+
+
+def IsInDir(file, dir):
+ try:
+ Path(file).relative_to(dir)
+ return True
+ except ValueError:
+ return False
+
+
+def GetVCSFilenameFromSrcdir(file, srcdir):
+ if srcdir not in Dumper.srcdirRepoInfo:
+ # Not in cache, so find it adnd cache it
+ if os.path.isdir(os.path.join(srcdir, ".hg")):
+ Dumper.srcdirRepoInfo[srcdir] = HGRepoInfo(srcdir)
+ else:
+ # Unknown VCS or file is not in a repo.
+ return None
+ return Dumper.srcdirRepoInfo[srcdir].GetFileInfo(file)
+
+
+def GetVCSFilename(file, srcdirs):
+ """Given a full path to a file, and the top source directory,
+ look for version control information about this file, and return
+ a tuple containing
+ 1) a specially formatted filename that contains the VCS type,
+ VCS location, relative filename, and revision number, formatted like:
+ vcs:vcs location:filename:revision
+ For example:
+ cvs:cvs.mozilla.org/cvsroot:mozilla/browser/app/nsBrowserApp.cpp:1.36
+ 2) the unmodified root information if it exists"""
+ (path, filename) = os.path.split(file)
+ if path == "" or filename == "":
+ return (file, None)
+
+ fileInfo = None
+ root = ""
+ if file in vcsFileInfoCache:
+ # Already cached this info, use it.
+ fileInfo = vcsFileInfoCache[file]
+ else:
+ for srcdir in srcdirs:
+ if not IsInDir(file, srcdir):
+ continue
+ fileInfo = GetVCSFilenameFromSrcdir(file, srcdir)
+ if fileInfo:
+ vcsFileInfoCache[file] = fileInfo
+ break
+
+ if fileInfo:
+ file = fileInfo.filename
+ root = fileInfo.root
+
+ # we want forward slashes on win32 paths
+ return (file.replace("\\", "/"), root)
+
+
+def validate_install_manifests(install_manifest_args):
+ args = []
+ for arg in install_manifest_args:
+ bits = arg.split(",")
+ if len(bits) != 2:
+ raise ValueError(
+ "Invalid format for --install-manifest: " "specify manifest,target_dir"
+ )
+ manifest_file, destination = [os.path.abspath(b) for b in bits]
+ if not os.path.isfile(manifest_file):
+ raise IOError(errno.ENOENT, "Manifest file not found", manifest_file)
+ if not os.path.isdir(destination):
+ raise IOError(errno.ENOENT, "Install directory not found", destination)
+ try:
+ manifest = InstallManifest(manifest_file)
+ except UnreadableInstallManifest:
+ raise IOError(errno.EINVAL, "Error parsing manifest file", manifest_file)
+ args.append((manifest, destination))
+ return args
+
+
+def make_file_mapping(install_manifests):
+ file_mapping = {}
+ for manifest, destination in install_manifests:
+ destination = os.path.abspath(destination)
+ reg = FileRegistry()
+ manifest.populate_registry(reg)
+ for dst, src in reg:
+ if hasattr(src, "path"):
+ # Any paths that get compared to source file names need to go through realpath.
+ abs_dest = realpath(os.path.join(destination, dst))
+ file_mapping[abs_dest] = realpath(src.path)
+ return file_mapping
+
+
+@memoize
+def get_generated_file_s3_path(filename, rel_path, bucket):
+ """Given a filename, return a path formatted similarly to
+ GetVCSFilename but representing a file available in an s3 bucket."""
+ with open(filename, "rb") as f:
+ path = get_filename_with_digest(rel_path, f.read())
+ return "s3:{bucket}:{path}:".format(bucket=bucket, path=path)
+
+
+def GetPlatformSpecificDumper(**kwargs):
+ """This function simply returns a instance of a subclass of Dumper
+ that is appropriate for the current platform."""
+ return {"WINNT": Dumper_Win32, "Linux": Dumper_Linux, "Darwin": Dumper_Mac}[
+ buildconfig.substs["OS_ARCH"]
+ ](**kwargs)
+
+
+def SourceIndex(fileStream, outputPath, vcs_root, s3_bucket):
+ """Takes a list of files, writes info to a data block in a .stream file"""
+ # Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing
+ # Create the srcsrv data block that indexes the pdb file
+ result = True
+ pdbStreamFile = open(outputPath, "w")
+ pdbStreamFile.write(
+ "SRCSRV: ini ------------------------------------------------\r\n"
+ + "VERSION=2\r\n"
+ + "INDEXVERSION=2\r\n"
+ + "VERCTRL=http\r\n"
+ + "SRCSRV: variables ------------------------------------------\r\n"
+ + "SRCSRVVERCTRL=http\r\n"
+ + "RUST_GITHUB_TARGET=https://github.com/rust-lang/rust/raw/%var4%/%var3%\r\n"
+ )
+ pdbStreamFile.write("HGSERVER=" + vcs_root + "\r\n")
+ pdbStreamFile.write("HG_TARGET=%hgserver%/raw-file/%var4%/%var3%\r\n")
+
+ if s3_bucket:
+ pdbStreamFile.write("S3_BUCKET=" + s3_bucket + "\r\n")
+ pdbStreamFile.write("S3_TARGET=https://%s3_bucket%.s3.amazonaws.com/%var3%\r\n")
+
+ # Allow each entry to choose its template via "var2".
+ # Possible values for var2 are: HG_TARGET / S3_TARGET / RUST_GITHUB_TARGET
+ pdbStreamFile.write("SRCSRVTRG=%fnvar%(%var2%)\r\n")
+
+ pdbStreamFile.write(
+ "SRCSRV: source files ---------------------------------------\r\n"
+ )
+ pdbStreamFile.write(fileStream)
+ pdbStreamFile.write(
+ "SRCSRV: end ------------------------------------------------\r\n\n"
+ )
+ pdbStreamFile.close()
+ return result
+
+
+class Dumper:
+ """This class can dump symbols from a file with debug info, and
+ store the output in a directory structure that is valid for use as
+ a Breakpad symbol server. Requires a path to a dump_syms binary--
+ |dump_syms| and a directory to store symbols in--|symbol_path|.
+ Optionally takes a list of processor architectures to process from
+ each debug file--|archs|, the full path to the top source
+ directory--|srcdir|, for generating relative source file names,
+ and an option to copy debug info files alongside the dumped
+ symbol files--|copy_debug|, mostly useful for creating a
+ Microsoft Symbol Server from the resulting output.
+
+ You don't want to use this directly if you intend to process files.
+ Instead, call GetPlatformSpecificDumper to get an instance of a
+ subclass."""
+
+ srcdirRepoInfo = {}
+
+ def __init__(
+ self,
+ dump_syms,
+ symbol_path,
+ archs=None,
+ srcdirs=[],
+ copy_debug=False,
+ vcsinfo=False,
+ srcsrv=False,
+ s3_bucket=None,
+ file_mapping=None,
+ ):
+ # popen likes absolute paths, at least on windows
+ self.dump_syms = os.path.abspath(dump_syms)
+ self.symbol_path = symbol_path
+ if archs is None:
+ # makes the loop logic simpler
+ self.archs = [""]
+ else:
+ self.archs = ["-a %s" % a for a in archs.split()]
+ # Any paths that get compared to source file names need to go through realpath.
+ self.srcdirs = [realpath(s) for s in srcdirs]
+ self.copy_debug = copy_debug
+ self.vcsinfo = vcsinfo
+ self.srcsrv = srcsrv
+ self.s3_bucket = s3_bucket
+ self.file_mapping = file_mapping or {}
+ # Add a static mapping for Rust sources. Since Rust 1.30 official Rust builds map
+ # source paths to start with "/rust/<sha>/".
+ rust_sha = buildconfig.substs["RUSTC_COMMIT"]
+ rust_srcdir = "/rustc/" + rust_sha
+ self.srcdirs.append(rust_srcdir)
+ Dumper.srcdirRepoInfo[rust_srcdir] = GitRepoInfo(
+ rust_srcdir, rust_sha, "https://github.com/rust-lang/rust/"
+ )
+
+ # subclasses override this
+ def ShouldProcess(self, file):
+ return True
+
+ # This is a no-op except on Win32
+ def SourceServerIndexing(
+ self, debug_file, guid, sourceFileStream, vcs_root, s3_bucket
+ ):
+ return ""
+
+ # subclasses override this if they want to support this
+ def CopyExeAndDebugInfo(self, file, debug_file, guid, code_file, code_id):
+ """This function will copy a library or executable and the file holding the
+ debug information to |symbol_path|"""
+ pass
+
+ def Process(self, file_to_process, count_ctors=False):
+ """Process the given file."""
+ if self.ShouldProcess(os.path.abspath(file_to_process)):
+ self.ProcessFile(file_to_process, count_ctors=count_ctors)
+
+ def ProcessFile(self, file, dsymbundle=None, count_ctors=False):
+ """Dump symbols from these files into a symbol file, stored
+ in the proper directory structure in |symbol_path|; processing is performed
+ asynchronously, and Finish must be called to wait for it complete and cleanup.
+ All files after the first are fallbacks in case the first file does not process
+ successfully; if it does, no other files will be touched."""
+ print("Beginning work for file: %s" % file, file=sys.stderr)
+
+ # tries to get the vcs root from the .mozconfig first - if it's not set
+ # the tinderbox vcs path will be assigned further down
+ vcs_root = os.environ.get("MOZ_SOURCE_REPO")
+ for arch_num, arch in enumerate(self.archs):
+ self.ProcessFileWork(
+ file, arch_num, arch, vcs_root, dsymbundle, count_ctors=count_ctors
+ )
+
+ def dump_syms_cmdline(self, file, arch, dsymbundle=None):
+ """
+ Get the commandline used to invoke dump_syms.
+ """
+ # The Mac dumper overrides this.
+ return [self.dump_syms, "--inlines", file]
+
+ def ProcessFileWork(
+ self, file, arch_num, arch, vcs_root, dsymbundle=None, count_ctors=False
+ ):
+ ctors = 0
+ t_start = time.time()
+ print("Processing file: %s" % file, file=sys.stderr)
+
+ sourceFileStream = ""
+ code_id, code_file = None, None
+ try:
+ cmd = self.dump_syms_cmdline(file, arch, dsymbundle=dsymbundle)
+ print(" ".join(cmd), file=sys.stderr)
+ proc = subprocess.Popen(
+ cmd,
+ universal_newlines=True,
+ stdout=subprocess.PIPE,
+ )
+ try:
+ module_line = next(proc.stdout)
+ except StopIteration:
+ module_line = ""
+ if module_line.startswith("MODULE"):
+ # MODULE os cpu guid debug_file
+ (guid, debug_file) = (module_line.split())[3:5]
+ # strip off .pdb extensions, and append .sym
+ sym_file = re.sub("\.pdb$", "", debug_file) + ".sym"
+ # we do want forward slashes here
+ rel_path = os.path.join(debug_file, guid, sym_file).replace("\\", "/")
+ full_path = os.path.normpath(os.path.join(self.symbol_path, rel_path))
+ try:
+ os.makedirs(os.path.dirname(full_path))
+ except OSError: # already exists
+ pass
+ f = open(full_path, "w")
+ f.write(module_line)
+ # now process the rest of the output
+ for line in proc.stdout:
+ if line.startswith("FILE"):
+ # FILE index filename
+ (x, index, filename) = line.rstrip().split(None, 2)
+ # We want original file paths for the source server.
+ sourcepath = filename
+ filename = realpath(filename)
+ if filename in self.file_mapping:
+ filename = self.file_mapping[filename]
+ if self.vcsinfo:
+ try:
+ gen_path = Path(filename)
+ rel_gen_path = gen_path.relative_to(
+ buildconfig.topobjdir
+ )
+ except ValueError:
+ gen_path = None
+ if (
+ gen_path
+ and gen_path.exists()
+ and gen_path.suffix in GENERATED_SOURCE_EXTS
+ and self.s3_bucket
+ ):
+ filename = get_generated_file_s3_path(
+ filename, str(rel_gen_path), self.s3_bucket
+ )
+ rootname = ""
+ else:
+ (filename, rootname) = GetVCSFilename(
+ filename, self.srcdirs
+ )
+ # sets vcs_root in case the loop through files were to end
+ # on an empty rootname
+ if vcs_root is None:
+ if rootname:
+ vcs_root = rootname
+ # Emit an entry for the file mapping for the srcsrv stream
+ if filename.startswith("hg:"):
+ (vcs, repo, source_file, revision) = filename.split(":", 3)
+ sourceFileStream += sourcepath + "*HG_TARGET*" + source_file
+ sourceFileStream += "*" + revision + "\r\n"
+ elif filename.startswith("s3:"):
+ (vcs, bucket, source_file, nothing) = filename.split(":", 3)
+ sourceFileStream += sourcepath + "*S3_TARGET*"
+ sourceFileStream += source_file + "\r\n"
+ elif filename.startswith("git:github.com/rust-lang/rust:"):
+ (vcs, repo, source_file, revision) = filename.split(":", 3)
+ sourceFileStream += sourcepath + "*RUST_GITHUB_TARGET*"
+ sourceFileStream += source_file + "*" + revision + "\r\n"
+ f.write("FILE %s %s\n" % (index, filename))
+ elif line.startswith("INFO CODE_ID "):
+ # INFO CODE_ID code_id code_file
+ # This gives some info we can use to
+ # store binaries in the symbol store.
+ bits = line.rstrip().split(None, 3)
+ if len(bits) == 4:
+ code_id, code_file = bits[2:]
+ f.write(line)
+ else:
+ if count_ctors and line.startswith("FUNC "):
+ # Static initializers, as created by clang and gcc
+ # have symbols that start with "_GLOBAL_sub"
+ if "_GLOBAL__sub_" in line:
+ ctors += 1
+ # MSVC creates `dynamic initializer for '...'`
+ # symbols.
+ elif "`dynamic initializer for '" in line:
+ ctors += 1
+
+ # pass through all other lines unchanged
+ f.write(line)
+ f.close()
+ retcode = proc.wait()
+ if retcode != 0:
+ raise RuntimeError(
+ "dump_syms failed with error code %d while processing %s\n"
+ % (retcode, file)
+ )
+ # we output relative paths so callers can get a list of what
+ # was generated
+ print(rel_path)
+ if self.srcsrv and vcs_root:
+ # add source server indexing to the pdb file
+ self.SourceServerIndexing(
+ debug_file, guid, sourceFileStream, vcs_root, self.s3_bucket
+ )
+ # only copy debug the first time if we have multiple architectures
+ if self.copy_debug and arch_num == 0:
+ self.CopyExeAndDebugInfo(file, debug_file, guid, code_file, code_id)
+ else:
+ # For some reason, we didn't see the MODULE line as the first
+ # line of output, this is strictly required so fail irrespective
+ # of the process' return code.
+ retcode = proc.wait()
+ message = [
+ "dump_syms failed to produce the expected output",
+ "file: %s" % file,
+ "return code: %d" % retcode,
+ "first line of output: %s" % module_line,
+ ]
+ raise RuntimeError("\n----------\n".join(message))
+ except Exception as e:
+ print("Unexpected error: %s" % str(e), file=sys.stderr)
+ raise
+
+ if dsymbundle:
+ shutil.rmtree(dsymbundle)
+
+ if count_ctors:
+ import json
+
+ perfherder_data = {
+ "framework": {"name": "build_metrics"},
+ "suites": [
+ {
+ "name": "compiler_metrics",
+ "subtests": [
+ {
+ "name": "num_static_constructors",
+ "value": ctors,
+ "alertChangeType": "absolute",
+ "alertThreshold": 3,
+ }
+ ],
+ }
+ ],
+ }
+ perfherder_extra_options = os.environ.get("PERFHERDER_EXTRA_OPTIONS", "")
+ for opt in perfherder_extra_options.split():
+ for suite in perfherder_data["suites"]:
+ if opt not in suite.get("extraOptions", []):
+ suite.setdefault("extraOptions", []).append(opt)
+
+ if "asan" not in perfherder_extra_options.lower():
+ print(
+ "PERFHERDER_DATA: %s" % json.dumps(perfherder_data), file=sys.stderr
+ )
+
+ elapsed = time.time() - t_start
+ print("Finished processing %s in %.2fs" % (file, elapsed), file=sys.stderr)
+
+
+# Platform-specific subclasses. For the most part, these just have
+# logic to determine what files to extract symbols from.
+
+
+def locate_pdb(path):
+ """Given a path to a binary, attempt to locate the matching pdb file with simple heuristics:
+ * Look for a pdb file with the same base name next to the binary
+ * Look for a pdb file with the same base name in the cwd
+
+ Returns the path to the pdb file if it exists, or None if it could not be located.
+ """
+ path, ext = os.path.splitext(path)
+ pdb = path + ".pdb"
+ if os.path.isfile(pdb):
+ return pdb
+ # If there's no pdb next to the file, see if there's a pdb with the same root name
+ # in the cwd. We build some binaries directly into dist/bin, but put the pdb files
+ # in the relative objdir, which is the cwd when running this script.
+ base = os.path.basename(pdb)
+ pdb = os.path.join(os.getcwd(), base)
+ if os.path.isfile(pdb):
+ return pdb
+ return None
+
+
+class Dumper_Win32(Dumper):
+ fixedFilenameCaseCache = {}
+
+ def ShouldProcess(self, file):
+ """This function will allow processing of exe or dll files that have pdb
+ files with the same base name next to them."""
+ if file.endswith(".exe") or file.endswith(".dll"):
+ if locate_pdb(file) is not None:
+ return True
+ return False
+
+ def CopyExeAndDebugInfo(self, file, debug_file, guid, code_file, code_id):
+ """This function will copy the executable or dll and pdb files to |symbol_path|"""
+ pdb_file = locate_pdb(file)
+
+ rel_path = os.path.join(debug_file, guid, debug_file).replace("\\", "/")
+ full_path = os.path.normpath(os.path.join(self.symbol_path, rel_path))
+ shutil.copyfile(pdb_file, full_path)
+ print(rel_path)
+
+ # Copy the binary file as well
+ if code_file and code_id:
+ full_code_path = os.path.join(os.path.dirname(file), code_file)
+ if os.path.exists(full_code_path):
+ rel_path = os.path.join(code_file, code_id, code_file).replace(
+ "\\", "/"
+ )
+ full_path = os.path.normpath(os.path.join(self.symbol_path, rel_path))
+ try:
+ os.makedirs(os.path.dirname(full_path))
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+ shutil.copyfile(full_code_path, full_path)
+ print(rel_path)
+
+ def SourceServerIndexing(
+ self, debug_file, guid, sourceFileStream, vcs_root, s3_bucket
+ ):
+ # Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing
+ streamFilename = debug_file + ".stream"
+ stream_output_path = os.path.abspath(streamFilename)
+ # Call SourceIndex to create the .stream file
+ result = SourceIndex(sourceFileStream, stream_output_path, vcs_root, s3_bucket)
+ if self.copy_debug:
+ pdbstr = buildconfig.substs["PDBSTR"]
+ wine = buildconfig.substs.get("WINE")
+ if wine:
+ cmd = [wine, pdbstr]
+ else:
+ cmd = [pdbstr]
+ subprocess.call(
+ cmd
+ + [
+ "-w",
+ "-p:" + os.path.basename(debug_file),
+ "-i:" + os.path.basename(streamFilename),
+ "-s:srcsrv",
+ ],
+ cwd=os.path.dirname(stream_output_path),
+ )
+ # clean up all the .stream files when done
+ os.remove(stream_output_path)
+ return result
+
+
+class Dumper_Linux(Dumper):
+ objcopy = os.environ["OBJCOPY"] if "OBJCOPY" in os.environ else "objcopy"
+
+ def ShouldProcess(self, file):
+ """This function will allow processing of files that are
+ executable, or end with the .so extension, and additionally
+ file(1) reports as being ELF files. It expects to find the file
+ command in PATH."""
+ if file.endswith(".so") or os.access(file, os.X_OK):
+ return executables.get_type(file) == executables.ELF
+ return False
+
+ def CopyExeAndDebugInfo(self, file, debug_file, guid, code_file, code_id):
+ # We want to strip out the debug info, and add a
+ # .gnu_debuglink section to the object, so the debugger can
+ # actually load our debug info later.
+ # In some odd cases, the object might already have an irrelevant
+ # .gnu_debuglink section, and objcopy doesn't want to add one in
+ # such cases, so we make it remove it any existing one first.
+ file_dbg = file + ".dbg"
+ if (
+ subprocess.call([self.objcopy, "--only-keep-debug", file, file_dbg]) == 0
+ and subprocess.call(
+ [
+ self.objcopy,
+ "--remove-section",
+ ".gnu_debuglink",
+ "--add-gnu-debuglink=%s" % file_dbg,
+ file,
+ ]
+ )
+ == 0
+ ):
+ rel_path = os.path.join(debug_file, guid, debug_file + ".dbg")
+ full_path = os.path.normpath(os.path.join(self.symbol_path, rel_path))
+ shutil.move(file_dbg, full_path)
+ print(rel_path)
+ else:
+ if os.path.isfile(file_dbg):
+ os.unlink(file_dbg)
+
+
+class Dumper_Solaris(Dumper):
+ def ShouldProcess(self, file):
+ """This function will allow processing of files that are
+ executable, or end with the .so extension, and additionally
+ file(1) reports as being ELF files. It expects to find the file
+ command in PATH."""
+ if file.endswith(".so") or os.access(file, os.X_OK):
+ return executables.get_type(file) == executables.ELF
+ return False
+
+
+class Dumper_Mac(Dumper):
+ def ShouldProcess(self, file):
+ """This function will allow processing of files that are
+ executable, or end with the .dylib extension, and additionally
+ file(1) reports as being Mach-O files. It expects to find the file
+ command in PATH."""
+ if file.endswith(".dylib") or os.access(file, os.X_OK):
+ return executables.get_type(file) == executables.MACHO
+ return False
+
+ def ProcessFile(self, file, count_ctors=False):
+ print("Starting Mac pre-processing on file: %s" % file, file=sys.stderr)
+ dsymbundle = self.GenerateDSYM(file)
+ if dsymbundle:
+ # kick off new jobs per-arch with our new list of files
+ Dumper.ProcessFile(
+ self, file, dsymbundle=dsymbundle, count_ctors=count_ctors
+ )
+
+ def dump_syms_cmdline(self, file, arch, dsymbundle=None):
+ """
+ Get the commandline used to invoke dump_syms.
+ """
+ # dump_syms wants the path to the original binary and the .dSYM
+ # in order to dump all the symbols.
+ if dsymbundle:
+ # This is the .dSYM bundle.
+ return (
+ [self.dump_syms]
+ + arch.split()
+ + ["--inlines", "-j", "2", dsymbundle, file]
+ )
+ return Dumper.dump_syms_cmdline(self, file, arch)
+
+ def GenerateDSYM(self, file):
+ """dump_syms on Mac needs to be run on a dSYM bundle produced
+ by dsymutil(1), so run dsymutil here and pass the bundle name
+ down to the superclass method instead."""
+ t_start = time.time()
+ print("Running Mac pre-processing on file: %s" % (file,), file=sys.stderr)
+
+ dsymbundle = file + ".dSYM"
+ if os.path.exists(dsymbundle):
+ shutil.rmtree(dsymbundle)
+ dsymutil = buildconfig.substs["DSYMUTIL"]
+ # dsymutil takes --arch=foo instead of -a foo like everything else
+ cmd = (
+ [dsymutil] + [a.replace("-a ", "--arch=") for a in self.archs if a] + [file]
+ )
+ print(" ".join(cmd), file=sys.stderr)
+
+ dsymutil_proc = subprocess.Popen(
+ cmd, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+ )
+ dsymout, dsymerr = dsymutil_proc.communicate()
+ if dsymutil_proc.returncode != 0:
+ raise RuntimeError("Error running dsymutil: %s" % dsymerr)
+
+ # Regular dsymutil won't produce a .dSYM for files without symbols.
+ if not os.path.exists(dsymbundle):
+ print("No symbols found in file: %s" % (file,), file=sys.stderr)
+ return False
+
+ # llvm-dsymutil will produce a .dSYM for files without symbols or
+ # debug information, but only sometimes will it warn you about this.
+ # We don't want to run dump_syms on such bundles, because asserts
+ # will fire in debug mode and who knows what will happen in release.
+ #
+ # So we check for the error message and bail if it appears. If it
+ # doesn't, we carefully check the bundled DWARF to see if dump_syms
+ # will be OK with it.
+ if "warning: no debug symbols in" in dsymerr:
+ print(dsymerr, file=sys.stderr)
+ return False
+
+ contents_dir = os.path.join(dsymbundle, "Contents", "Resources", "DWARF")
+ if not os.path.exists(contents_dir):
+ print(
+ "No DWARF information in .dSYM bundle %s" % (dsymbundle,),
+ file=sys.stderr,
+ )
+ return False
+
+ files = os.listdir(contents_dir)
+ if len(files) != 1:
+ print("Unexpected files in .dSYM bundle %s" % (files,), file=sys.stderr)
+ return False
+
+ otool_out = subprocess.check_output(
+ [buildconfig.substs["OTOOL"], "-l", os.path.join(contents_dir, files[0])],
+ universal_newlines=True,
+ )
+ if "sectname __debug_info" not in otool_out:
+ print("No symbols in .dSYM bundle %s" % (dsymbundle,), file=sys.stderr)
+ return False
+
+ elapsed = time.time() - t_start
+ print("Finished processing %s in %.2fs" % (file, elapsed), file=sys.stderr)
+ return dsymbundle
+
+ def CopyExeAndDebugInfo(self, file, debug_file, guid, code_file, code_id):
+ """ProcessFile has already produced a dSYM bundle, so we should just
+ copy that to the destination directory. However, we'll package it
+ into a .tar because it's a bundle, so it's a directory. |file| here is
+ the original filename."""
+ dsymbundle = file + ".dSYM"
+ rel_path = os.path.join(debug_file, guid, os.path.basename(dsymbundle) + ".tar")
+ full_path = os.path.abspath(os.path.join(self.symbol_path, rel_path))
+ success = subprocess.call(
+ ["tar", "cf", full_path, os.path.basename(dsymbundle)],
+ cwd=os.path.dirname(dsymbundle),
+ stdout=open(os.devnull, "w"),
+ stderr=subprocess.STDOUT,
+ )
+ if success == 0 and os.path.exists(full_path):
+ print(rel_path)
+
+
+# Entry point if called as a standalone program
+
+
+def main():
+ parser = OptionParser(
+ usage="usage: %prog [options] <dump_syms binary> <symbol store path> <debug info files>"
+ )
+ parser.add_option(
+ "-c",
+ "--copy",
+ action="store_true",
+ dest="copy_debug",
+ default=False,
+ help="Copy debug info files into the same directory structure as symbol files",
+ )
+ parser.add_option(
+ "-a",
+ "--archs",
+ action="store",
+ dest="archs",
+ help="Run dump_syms -a <arch> for each space separated"
+ + "cpu architecture in ARCHS (only on OS X)",
+ )
+ parser.add_option(
+ "-s",
+ "--srcdir",
+ action="append",
+ dest="srcdir",
+ default=[],
+ help="Use SRCDIR to determine relative paths to source files",
+ )
+ parser.add_option(
+ "-v",
+ "--vcs-info",
+ action="store_true",
+ dest="vcsinfo",
+ help="Try to retrieve VCS info for each FILE listed in the output",
+ )
+ parser.add_option(
+ "-i",
+ "--source-index",
+ action="store_true",
+ dest="srcsrv",
+ default=False,
+ help="Add source index information to debug files, making them suitable"
+ + " for use in a source server.",
+ )
+ parser.add_option(
+ "--install-manifest",
+ action="append",
+ dest="install_manifests",
+ default=[],
+ help="""Use this install manifest to map filenames back
+to canonical locations in the source repository. Specify
+<install manifest filename>,<install destination> as a comma-separated pair.""",
+ )
+ parser.add_option(
+ "--count-ctors",
+ action="store_true",
+ dest="count_ctors",
+ default=False,
+ help="Count static initializers",
+ )
+ (options, args) = parser.parse_args()
+
+ # check to see if the pdbstr.exe exists
+ if options.srcsrv:
+ if "PDBSTR" not in buildconfig.substs:
+ print("pdbstr was not found by configure.\n", file=sys.stderr)
+ sys.exit(1)
+
+ if len(args) < 3:
+ parser.error("not enough arguments")
+ exit(1)
+
+ try:
+ manifests = validate_install_manifests(options.install_manifests)
+ except (IOError, ValueError) as e:
+ parser.error(str(e))
+ exit(1)
+ file_mapping = make_file_mapping(manifests)
+ _, bucket = get_s3_region_and_bucket()
+ dumper = GetPlatformSpecificDumper(
+ dump_syms=args[0],
+ symbol_path=args[1],
+ copy_debug=options.copy_debug,
+ archs=options.archs,
+ srcdirs=options.srcdir,
+ vcsinfo=options.vcsinfo,
+ srcsrv=options.srcsrv,
+ s3_bucket=bucket,
+ file_mapping=file_mapping,
+ )
+
+ dumper.Process(args[2], options.count_ctors)
+
+
+# run main if run directly
+if __name__ == "__main__":
+ main()