Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1096 insertions, 0 deletions

diff --git a/toolkit/crashreporter/tools/symbolstore.py b/toolkit/crashreporter/tools/symbolstore.py
new file mode 100755
index 0000000000..5dd5570a84
--- /dev/null
+++ b/toolkit/crashreporter/tools/symbolstore.py
@@ -0,0 +1,1096 @@
+#!/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Usage: symbolstore.py <params> <dump_syms path> <symbol store path>
+#                                <debug info files or dirs>
+#   Runs dump_syms on each debug info file specified on the command line,
+#   then places the resulting symbol file in the proper directory
+#   structure in the symbol store path.  Accepts multiple files
+#   on the command line, so can be called as part of a pipe using
+#   find <dir> | xargs symbolstore.pl <dump_syms> <storepath>
+#   But really, you might just want to pass it <dir>.
+#
+#   Parameters accepted:
+#     -c           : Copy debug info files to the same directory structure
+#                    as sym files. On Windows, this will also copy
+#                    binaries into the symbol store.
+#     -a "<archs>" : Run dump_syms -a <arch> for each space separated
+#                    cpu architecture in <archs> (only on OS X)
+#     -s <srcdir>  : Use <srcdir> as the top source directory to
+#                    generate relative filenames.
+
+import ctypes
+import errno
+import os
+import platform
+import re
+import shutil
+import subprocess
+import sys
+import textwrap
+import time
+from optparse import OptionParser
+from pathlib import Path
+
+import buildconfig
+from mozbuild.generated_sources import (
+    GENERATED_SOURCE_EXTS,
+    get_filename_with_digest,
+    get_s3_region_and_bucket,
+)
+from mozbuild.util import memoize
+from mozpack import executables
+from mozpack.copier import FileRegistry
+from mozpack.manifests import InstallManifest, UnreadableInstallManifest
+
+# Utility classes
+
+
+class VCSFileInfo:
+    """A base class for version-controlled file information. Ensures that the
+    following attributes are generated only once (successfully):
+
+        self.root
+        self.clean_root
+        self.revision
+        self.filename
+
+    The attributes are generated by a single call to the GetRoot,
+    GetRevision, and GetFilename methods. Those methods are explicitly not
+    implemented here and must be implemented in derived classes."""
+
+    def __init__(self, file):
+        if not file:
+            raise ValueError
+        self.file = file
+
+    def __getattr__(self, name):
+        """__getattr__ is only called for attributes that are not set on self,
+        so setting self.[attr] will prevent future calls to the GetRoot,
+        GetRevision, and GetFilename methods. We don't set the values on
+        failure on the off chance that a future call might succeed."""
+
+        if name == "root":
+            root = self.GetRoot()
+            if root:
+                self.root = root
+            return root
+
+        elif name == "clean_root":
+            clean_root = self.GetCleanRoot()
+            if clean_root:
+                self.clean_root = clean_root
+            return clean_root
+
+        elif name == "revision":
+            revision = self.GetRevision()
+            if revision:
+                self.revision = revision
+            return revision
+
+        elif name == "filename":
+            filename = self.GetFilename()
+            if filename:
+                self.filename = filename
+            return filename
+
+        raise AttributeError
+
+    def GetRoot(self):
+        """This method should return the unmodified root for the file or 'None'
+        on failure."""
+        raise NotImplementedError
+
+    def GetCleanRoot(self):
+        """This method should return the repository root for the file or 'None'
+        on failure."""
+        raise NotImplementedError
+
+    def GetRevision(self):
+        """This method should return the revision number for the file or 'None'
+        on failure."""
+        raise NotImplementedError
+
+    def GetFilename(self):
+        """This method should return the repository-specific filename for the
+        file or 'None' on failure."""
+        raise NotImplementedError
+
+
+# This regex separates protocol and optional username/password from a url.
+# For instance, all the following urls will be transformed into
+# 'foo.com/bar':
+#
+#   http://foo.com/bar
+#   svn+ssh://user@foo.com/bar
+#   svn+ssh://user:pass@foo.com/bar
+#
+rootRegex = re.compile(r"^\S+?:/+(?:[^\s/]*@)?(\S+)$")
+
+
+def read_output(*args):
+    (stdout, _) = subprocess.Popen(
+        args=args, universal_newlines=True, stdout=subprocess.PIPE
+    ).communicate()
+    return stdout.rstrip()
+
+
+class HGRepoInfo:
+    def __init__(self, path):
+        self.path = path
+
+        rev = os.environ.get("MOZ_SOURCE_CHANGESET")
+        if not rev:
+            rev = read_output("hg", "-R", path, "parent", "--template={node}")
+
+        # Look for the default hg path. If MOZ_SOURCE_REPO is set, we
+        # don't bother asking hg.
+        hg_root = os.environ.get("MOZ_SOURCE_REPO")
+        if hg_root:
+            root = hg_root
+        else:
+            root = read_output("hg", "-R", path, "showconfig", "paths.default")
+            if not root:
+                print("Failed to get HG Repo for %s" % path, file=sys.stderr)
+        cleanroot = None
+        if root:
+            match = rootRegex.match(root)
+            if match:
+                cleanroot = match.group(1)
+                if cleanroot.endswith("/"):
+                    cleanroot = cleanroot[:-1]
+        if cleanroot is None:
+            print(
+                textwrap.dedent(
+                    """\
+            Could not determine repo info for %s.  This is either not a clone of the web-based
+            repository, or you have not specified MOZ_SOURCE_REPO, or the clone is corrupt."""
+                )
+                % path,
+                sys.stderr,
+            )
+            sys.exit(1)
+        self.rev = rev
+        self.root = root
+        self.cleanroot = cleanroot
+
+    def GetFileInfo(self, file):
+        return HGFileInfo(file, self)
+
+
+class HGFileInfo(VCSFileInfo):
+    def __init__(self, file, repo):
+        VCSFileInfo.__init__(self, file)
+        self.repo = repo
+        self.file = os.path.relpath(file, repo.path)
+
+    def GetRoot(self):
+        return self.repo.root
+
+    def GetCleanRoot(self):
+        return self.repo.cleanroot
+
+    def GetRevision(self):
+        return self.repo.rev
+
+    def GetFilename(self):
+        if self.revision and self.clean_root:
+            return "hg:%s:%s:%s" % (self.clean_root, self.file, self.revision)
+        return self.file
+
+
+class GitRepoInfo:
+    """
+    Info about a local git repository. Does not currently
+    support discovering info about a git clone, the info must be
+    provided out-of-band.
+    """
+
+    def __init__(self, path, rev, root):
+        self.path = path
+        cleanroot = None
+        if root:
+            match = rootRegex.match(root)
+            if match:
+                cleanroot = match.group(1)
+                if cleanroot.endswith("/"):
+                    cleanroot = cleanroot[:-1]
+        if cleanroot is None:
+            print(
+                textwrap.dedent(
+                    """\
+            Could not determine repo info for %s (%s).  This is either not a clone of a web-based
+            repository, or you have not specified MOZ_SOURCE_REPO, or the clone is corrupt."""
+                )
+                % (path, root),
+                file=sys.stderr,
+            )
+            sys.exit(1)
+        self.rev = rev
+        self.cleanroot = cleanroot
+
+    def GetFileInfo(self, file):
+        return GitFileInfo(file, self)
+
+
+class GitFileInfo(VCSFileInfo):
+    def __init__(self, file, repo):
+        VCSFileInfo.__init__(self, file)
+        self.repo = repo
+        self.file = os.path.relpath(file, repo.path)
+
+    def GetRoot(self):
+        return self.repo.path
+
+    def GetCleanRoot(self):
+        return self.repo.cleanroot
+
+    def GetRevision(self):
+        return self.repo.rev
+
+    def GetFilename(self):
+        if self.revision and self.clean_root:
+            return "git:%s:%s:%s" % (self.clean_root, self.file, self.revision)
+        return self.file
+
+
+# Utility functions
+
+
+# A cache of files for which VCS info has already been determined. Used to
+# prevent extra filesystem activity or process launching.
+vcsFileInfoCache = {}
+
+if platform.system() == "Windows":
+
+    def realpath(path):
+        """
+        Normalize a path using `GetFinalPathNameByHandleW` to get the
+        path with all components in the case they exist in on-disk, so
+        that making links to a case-sensitive server (hg.mozilla.org) works.
+
+        This function also resolves any symlinks in the path.
+        """
+        # Return the original path if something fails, which can happen for paths that
+        # don't exist on this system (like paths from the CRT).
+        result = path
+
+        ctypes.windll.kernel32.SetErrorMode(ctypes.c_uint(1))
+        handle = ctypes.windll.kernel32.CreateFileW(
+            path,
+            # GENERIC_READ
+            0x80000000,
+            # FILE_SHARE_READ
+            1,
+            None,
+            # OPEN_EXISTING
+            3,
+            # FILE_FLAG_BACKUP_SEMANTICS
+            # This is necessary to open
+            # directory handles.
+            0x02000000,
+            None,
+        )
+        if handle != -1:
+            size = ctypes.windll.kernel32.GetFinalPathNameByHandleW(handle, None, 0, 0)
+            buf = ctypes.create_unicode_buffer(size)
+            if (
+                ctypes.windll.kernel32.GetFinalPathNameByHandleW(handle, buf, size, 0)
+                > 0
+            ):
+                # The return value of GetFinalPathNameByHandleW uses the
+                # '\\?\' prefix.
+                result = buf.value[4:]
+            ctypes.windll.kernel32.CloseHandle(handle)
+        return result
+
+
+else:
+    # Just use the os.path version otherwise.
+    realpath = os.path.realpath
+
+
+def IsInDir(file, dir):
+    try:
+        Path(file).relative_to(dir)
+        return True
+    except ValueError:
+        return False
+
+
+def GetVCSFilenameFromSrcdir(file, srcdir):
+    if srcdir not in Dumper.srcdirRepoInfo:
+        # Not in cache, so find it adnd cache it
+        if os.path.isdir(os.path.join(srcdir, ".hg")):
+            Dumper.srcdirRepoInfo[srcdir] = HGRepoInfo(srcdir)
+        else:
+            # Unknown VCS or file is not in a repo.
+            return None
+    return Dumper.srcdirRepoInfo[srcdir].GetFileInfo(file)
+
+
+def GetVCSFilename(file, srcdirs):
+    """Given a full path to a file, and the top source directory,
+    look for version control information about this file, and return
+    a tuple containing
+    1) a specially formatted filename that contains the VCS type,
+    VCS location, relative filename, and revision number, formatted like:
+    vcs:vcs location:filename:revision
+    For example:
+    cvs:cvs.mozilla.org/cvsroot:mozilla/browser/app/nsBrowserApp.cpp:1.36
+    2) the unmodified root information if it exists"""
+    (path, filename) = os.path.split(file)
+    if path == "" or filename == "":
+        return (file, None)
+
+    fileInfo = None
+    root = ""
+    if file in vcsFileInfoCache:
+        # Already cached this info, use it.
+        fileInfo = vcsFileInfoCache[file]
+    else:
+        for srcdir in srcdirs:
+            if not IsInDir(file, srcdir):
+                continue
+            fileInfo = GetVCSFilenameFromSrcdir(file, srcdir)
+            if fileInfo:
+                vcsFileInfoCache[file] = fileInfo
+                break
+
+    if fileInfo:
+        file = fileInfo.filename
+        root = fileInfo.root
+
+    # we want forward slashes on win32 paths
+    return (file.replace("\\", "/"), root)
+
+
+def validate_install_manifests(install_manifest_args):
+    args = []
+    for arg in install_manifest_args:
+        bits = arg.split(",")
+        if len(bits) != 2:
+            raise ValueError(
+                "Invalid format for --install-manifest: " "specify manifest,target_dir"
+            )
+        manifest_file, destination = [os.path.abspath(b) for b in bits]
+        if not os.path.isfile(manifest_file):
+            raise IOError(errno.ENOENT, "Manifest file not found", manifest_file)
+        if not os.path.isdir(destination):
+            raise IOError(errno.ENOENT, "Install directory not found", destination)
+        try:
+            manifest = InstallManifest(manifest_file)
+        except UnreadableInstallManifest:
+            raise IOError(errno.EINVAL, "Error parsing manifest file", manifest_file)
+        args.append((manifest, destination))
+    return args
+
+
+def make_file_mapping(install_manifests):
+    file_mapping = {}
+    for manifest, destination in install_manifests:
+        destination = os.path.abspath(destination)
+        reg = FileRegistry()
+        manifest.populate_registry(reg)
+        for dst, src in reg:
+            if hasattr(src, "path"):
+                # Any paths that get compared to source file names need to go through realpath.
+                abs_dest = realpath(os.path.join(destination, dst))
+                file_mapping[abs_dest] = realpath(src.path)
+    return file_mapping
+
+
+@memoize
+def get_generated_file_s3_path(filename, rel_path, bucket):
+    """Given a filename, return a path formatted similarly to
+    GetVCSFilename but representing a file available in an s3 bucket."""
+    with open(filename, "rb") as f:
+        path = get_filename_with_digest(rel_path, f.read())
+        return "s3:{bucket}:{path}:".format(bucket=bucket, path=path)
+
+
+def GetPlatformSpecificDumper(**kwargs):
+    """This function simply returns a instance of a subclass of Dumper
+    that is appropriate for the current platform."""
+    return {"WINNT": Dumper_Win32, "Linux": Dumper_Linux, "Darwin": Dumper_Mac}[
+        buildconfig.substs["OS_ARCH"]
+    ](**kwargs)
+
+
+def SourceIndex(fileStream, outputPath, vcs_root, s3_bucket):
+    """Takes a list of files, writes info to a data block in a .stream file"""
+    # Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing
+    # Create the srcsrv data block that indexes the pdb file
+    result = True
+    pdbStreamFile = open(outputPath, "w")
+    pdbStreamFile.write(
+        "SRCSRV: ini ------------------------------------------------\r\n"
+        + "VERSION=2\r\n"
+        + "INDEXVERSION=2\r\n"
+        + "VERCTRL=http\r\n"
+        + "SRCSRV: variables ------------------------------------------\r\n"
+        + "SRCSRVVERCTRL=http\r\n"
+        + "RUST_GITHUB_TARGET=https://github.com/rust-lang/rust/raw/%var4%/%var3%\r\n"
+    )
+    pdbStreamFile.write("HGSERVER=" + vcs_root + "\r\n")
+    pdbStreamFile.write("HG_TARGET=%hgserver%/raw-file/%var4%/%var3%\r\n")
+
+    if s3_bucket:
+        pdbStreamFile.write("S3_BUCKET=" + s3_bucket + "\r\n")
+        pdbStreamFile.write("S3_TARGET=https://%s3_bucket%.s3.amazonaws.com/%var3%\r\n")
+
+    # Allow each entry to choose its template via "var2".
+    # Possible values for var2 are: HG_TARGET / S3_TARGET / RUST_GITHUB_TARGET
+    pdbStreamFile.write("SRCSRVTRG=%fnvar%(%var2%)\r\n")
+
+    pdbStreamFile.write(
+        "SRCSRV: source files ---------------------------------------\r\n"
+    )
+    pdbStreamFile.write(fileStream)
+    pdbStreamFile.write(
+        "SRCSRV: end ------------------------------------------------\r\n\n"
+    )
+    pdbStreamFile.close()
+    return result
+
+
+class Dumper:
+    """This class can dump symbols from a file with debug info, and
+    store the output in a directory structure that is valid for use as
+    a Breakpad symbol server.  Requires a path to a dump_syms binary--
+    |dump_syms| and a directory to store symbols in--|symbol_path|.
+    Optionally takes a list of processor architectures to process from
+    each debug file--|archs|, the full path to the top source
+    directory--|srcdir|, for generating relative source file names,
+    and an option to copy debug info files alongside the dumped
+    symbol files--|copy_debug|, mostly useful for creating a
+    Microsoft Symbol Server from the resulting output.
+
+    You don't want to use this directly if you intend to process files.
+    Instead, call GetPlatformSpecificDumper to get an instance of a
+    subclass."""
+
+    srcdirRepoInfo = {}
+
+    def __init__(
+        self,
+        dump_syms,
+        symbol_path,
+        archs=None,
+        srcdirs=[],
+        copy_debug=False,
+        vcsinfo=False,
+        srcsrv=False,
+        s3_bucket=None,
+        file_mapping=None,
+    ):
+        # popen likes absolute paths, at least on windows
+        self.dump_syms = os.path.abspath(dump_syms)
+        self.symbol_path = symbol_path
+        if archs is None:
+            # makes the loop logic simpler
+            self.archs = [""]
+        else:
+            self.archs = ["-a %s" % a for a in archs.split()]
+        # Any paths that get compared to source file names need to go through realpath.
+        self.srcdirs = [realpath(s) for s in srcdirs]
+        self.copy_debug = copy_debug
+        self.vcsinfo = vcsinfo
+        self.srcsrv = srcsrv
+        self.s3_bucket = s3_bucket
+        self.file_mapping = file_mapping or {}
+        # Add a static mapping for Rust sources. Since Rust 1.30 official Rust builds map
+        # source paths to start with "/rust/<sha>/".
+        rust_sha = buildconfig.substs["RUSTC_COMMIT"]
+        rust_srcdir = "/rustc/" + rust_sha
+        self.srcdirs.append(rust_srcdir)
+        Dumper.srcdirRepoInfo[rust_srcdir] = GitRepoInfo(
+            rust_srcdir, rust_sha, "https://github.com/rust-lang/rust/"
+        )
+
+    # subclasses override this
+    def ShouldProcess(self, file):
+        return True
+
+    # This is a no-op except on Win32
+    def SourceServerIndexing(
+        self, debug_file, guid, sourceFileStream, vcs_root, s3_bucket
+    ):
+        return ""
+
+    # subclasses override this if they want to support this
+    def CopyExeAndDebugInfo(self, file, debug_file, guid, code_file, code_id):
+        """This function will copy a library or executable and the file holding the
+        debug information to |symbol_path|"""
+        pass
+
+    def Process(self, file_to_process, count_ctors=False):
+        """Process the given file."""
+        if self.ShouldProcess(os.path.abspath(file_to_process)):
+            self.ProcessFile(file_to_process, count_ctors=count_ctors)
+
+    def ProcessFile(self, file, dsymbundle=None, count_ctors=False):
+        """Dump symbols from these files into a symbol file, stored
+        in the proper directory structure in  |symbol_path|; processing is performed
+        asynchronously, and Finish must be called to wait for it complete and cleanup.
+        All files after the first are fallbacks in case the first file does not process
+        successfully; if it does, no other files will be touched."""
+        print("Beginning work for file: %s" % file, file=sys.stderr)
+
+        # tries to get the vcs root from the .mozconfig first - if it's not set
+        # the tinderbox vcs path will be assigned further down
+        vcs_root = os.environ.get("MOZ_SOURCE_REPO")
+        for arch_num, arch in enumerate(self.archs):
+            self.ProcessFileWork(
+                file, arch_num, arch, vcs_root, dsymbundle, count_ctors=count_ctors
+            )
+
+    def dump_syms_cmdline(self, file, arch, dsymbundle=None):
+        """
+        Get the commandline used to invoke dump_syms.
+        """
+        # The Mac dumper overrides this.
+        return [self.dump_syms, "--inlines", file]
+
+    def ProcessFileWork(
+        self, file, arch_num, arch, vcs_root, dsymbundle=None, count_ctors=False
+    ):
+        ctors = 0
+        t_start = time.time()
+        print("Processing file: %s" % file, file=sys.stderr)
+
+        sourceFileStream = ""
+        code_id, code_file = None, None
+        try:
+            cmd = self.dump_syms_cmdline(file, arch, dsymbundle=dsymbundle)
+            print(" ".join(cmd), file=sys.stderr)
+            proc = subprocess.Popen(
+                cmd,
+                universal_newlines=True,
+                stdout=subprocess.PIPE,
+            )
+            try:
+                module_line = next(proc.stdout)
+            except StopIteration:
+                module_line = ""
+            if module_line.startswith("MODULE"):
+                # MODULE os cpu guid debug_file
+                (guid, debug_file) = (module_line.split())[3:5]
+                # strip off .pdb extensions, and append .sym
+                sym_file = re.sub("\.pdb$", "", debug_file) + ".sym"
+                # we do want forward slashes here
+                rel_path = os.path.join(debug_file, guid, sym_file).replace("\\", "/")
+                full_path = os.path.normpath(os.path.join(self.symbol_path, rel_path))
+                try:
+                    os.makedirs(os.path.dirname(full_path))
+                except OSError:  # already exists
+                    pass
+                f = open(full_path, "w")
+                f.write(module_line)
+                # now process the rest of the output
+                for line in proc.stdout:
+                    if line.startswith("FILE"):
+                        # FILE index filename
+                        (x, index, filename) = line.rstrip().split(None, 2)
+                        # We want original file paths for the source server.
+                        sourcepath = filename
+                        filename = realpath(filename)
+                        if filename in self.file_mapping:
+                            filename = self.file_mapping[filename]
+                        if self.vcsinfo:
+                            try:
+                                gen_path = Path(filename)
+                                rel_gen_path = gen_path.relative_to(
+                                    buildconfig.topobjdir
+                                )
+                            except ValueError:
+                                gen_path = None
+                            if (
+                                gen_path
+                                and gen_path.exists()
+                                and gen_path.suffix in GENERATED_SOURCE_EXTS
+                                and self.s3_bucket
+                            ):
+                                filename = get_generated_file_s3_path(
+                                    filename, str(rel_gen_path), self.s3_bucket
+                                )
+                                rootname = ""
+                            else:
+                                (filename, rootname) = GetVCSFilename(
+                                    filename, self.srcdirs
+                                )
+                            # sets vcs_root in case the loop through files were to end
+                            # on an empty rootname
+                            if vcs_root is None:
+                                if rootname:
+                                    vcs_root = rootname
+                        # Emit an entry for the file mapping for the srcsrv stream
+                        if filename.startswith("hg:"):
+                            (vcs, repo, source_file, revision) = filename.split(":", 3)
+                            sourceFileStream += sourcepath + "*HG_TARGET*" + source_file
+                            sourceFileStream += "*" + revision + "\r\n"
+                        elif filename.startswith("s3:"):
+                            (vcs, bucket, source_file, nothing) = filename.split(":", 3)
+                            sourceFileStream += sourcepath + "*S3_TARGET*"
+                            sourceFileStream += source_file + "\r\n"
+                        elif filename.startswith("git:github.com/rust-lang/rust:"):
+                            (vcs, repo, source_file, revision) = filename.split(":", 3)
+                            sourceFileStream += sourcepath + "*RUST_GITHUB_TARGET*"
+                            sourceFileStream += source_file + "*" + revision + "\r\n"
+                        f.write("FILE %s %s\n" % (index, filename))
+                    elif line.startswith("INFO CODE_ID "):
+                        # INFO CODE_ID code_id code_file
+                        # This gives some info we can use to
+                        # store binaries in the symbol store.
+                        bits = line.rstrip().split(None, 3)
+                        if len(bits) == 4:
+                            code_id, code_file = bits[2:]
+                        f.write(line)
+                    else:
+                        if count_ctors and line.startswith("FUNC "):
+                            # Static initializers, as created by clang and gcc
+                            # have symbols that start with "_GLOBAL_sub"
+                            if "_GLOBAL__sub_" in line:
+                                ctors += 1
+                            # MSVC creates `dynamic initializer for '...'`
+                            # symbols.
+                            elif "`dynamic initializer for '" in line:
+                                ctors += 1
+
+                        # pass through all other lines unchanged
+                        f.write(line)
+                f.close()
+                retcode = proc.wait()
+                if retcode != 0:
+                    raise RuntimeError(
+                        "dump_syms failed with error code %d while processing %s\n"
+                        % (retcode, file)
+                    )
+                # we output relative paths so callers can get a list of what
+                # was generated
+                print(rel_path)
+                if self.srcsrv and vcs_root:
+                    # add source server indexing to the pdb file
+                    self.SourceServerIndexing(
+                        debug_file, guid, sourceFileStream, vcs_root, self.s3_bucket
+                    )
+                # only copy debug the first time if we have multiple architectures
+                if self.copy_debug and arch_num == 0:
+                    self.CopyExeAndDebugInfo(file, debug_file, guid, code_file, code_id)
+            else:
+                # For some reason, we didn't see the MODULE line as the first
+                # line of output, this is strictly required so fail irrespective
+                # of the process' return code.
+                retcode = proc.wait()
+                message = [
+                    "dump_syms failed to produce the expected output",
+                    "file: %s" % file,
+                    "return code: %d" % retcode,
+                    "first line of output: %s" % module_line,
+                ]
+                raise RuntimeError("\n----------\n".join(message))
+        except Exception as e:
+            print("Unexpected error: %s" % str(e), file=sys.stderr)
+            raise
+
+        if dsymbundle:
+            shutil.rmtree(dsymbundle)
+
+        if count_ctors:
+            import json
+
+            perfherder_data = {
+                "framework": {"name": "build_metrics"},
+                "suites": [
+                    {
+                        "name": "compiler_metrics",
+                        "subtests": [
+                            {
+                                "name": "num_static_constructors",
+                                "value": ctors,
+                                "alertChangeType": "absolute",
+                                "alertThreshold": 3,
+                            }
+                        ],
+                    }
+                ],
+            }
+            perfherder_extra_options = os.environ.get("PERFHERDER_EXTRA_OPTIONS", "")
+            for opt in perfherder_extra_options.split():
+                for suite in perfherder_data["suites"]:
+                    if opt not in suite.get("extraOptions", []):
+                        suite.setdefault("extraOptions", []).append(opt)
+
+            if "asan" not in perfherder_extra_options.lower():
+                print(
+                    "PERFHERDER_DATA: %s" % json.dumps(perfherder_data), file=sys.stderr
+                )
+
+        elapsed = time.time() - t_start
+        print("Finished processing %s in %.2fs" % (file, elapsed), file=sys.stderr)
+
+
+# Platform-specific subclasses.  For the most part, these just have
+# logic to determine what files to extract symbols from.
+
+
+def locate_pdb(path):
+    """Given a path to a binary, attempt to locate the matching pdb file with simple heuristics:
+    * Look for a pdb file with the same base name next to the binary
+    * Look for a pdb file with the same base name in the cwd
+
+    Returns the path to the pdb file if it exists, or None if it could not be located.
+    """
+    path, ext = os.path.splitext(path)
+    pdb = path + ".pdb"
+    if os.path.isfile(pdb):
+        return pdb
+    # If there's no pdb next to the file, see if there's a pdb with the same root name
+    # in the cwd. We build some binaries directly into dist/bin, but put the pdb files
+    # in the relative objdir, which is the cwd when running this script.
+    base = os.path.basename(pdb)
+    pdb = os.path.join(os.getcwd(), base)
+    if os.path.isfile(pdb):
+        return pdb
+    return None
+
+
+class Dumper_Win32(Dumper):
+    fixedFilenameCaseCache = {}
+
+    def ShouldProcess(self, file):
+        """This function will allow processing of exe or dll files that have pdb
+        files with the same base name next to them."""
+        if file.endswith(".exe") or file.endswith(".dll"):
+            if locate_pdb(file) is not None:
+                return True
+        return False
+
+    def CopyExeAndDebugInfo(self, file, debug_file, guid, code_file, code_id):
+        """This function will copy the executable or dll and pdb files to |symbol_path|"""
+        pdb_file = locate_pdb(file)
+
+        rel_path = os.path.join(debug_file, guid, debug_file).replace("\\", "/")
+        full_path = os.path.normpath(os.path.join(self.symbol_path, rel_path))
+        shutil.copyfile(pdb_file, full_path)
+        print(rel_path)
+
+        # Copy the binary file as well
+        if code_file and code_id:
+            full_code_path = os.path.join(os.path.dirname(file), code_file)
+            if os.path.exists(full_code_path):
+                rel_path = os.path.join(code_file, code_id, code_file).replace(
+                    "\\", "/"
+                )
+                full_path = os.path.normpath(os.path.join(self.symbol_path, rel_path))
+                try:
+                    os.makedirs(os.path.dirname(full_path))
+                except OSError as e:
+                    if e.errno != errno.EEXIST:
+                        raise
+                shutil.copyfile(full_code_path, full_path)
+                print(rel_path)
+
+    def SourceServerIndexing(
+        self, debug_file, guid, sourceFileStream, vcs_root, s3_bucket
+    ):
+        # Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing
+        streamFilename = debug_file + ".stream"
+        stream_output_path = os.path.abspath(streamFilename)
+        # Call SourceIndex to create the .stream file
+        result = SourceIndex(sourceFileStream, stream_output_path, vcs_root, s3_bucket)
+        if self.copy_debug:
+            pdbstr = buildconfig.substs["PDBSTR"]
+            wine = buildconfig.substs.get("WINE")
+            if wine:
+                cmd = [wine, pdbstr]
+            else:
+                cmd = [pdbstr]
+            subprocess.call(
+                cmd
+                + [
+                    "-w",
+                    "-p:" + os.path.basename(debug_file),
+                    "-i:" + os.path.basename(streamFilename),
+                    "-s:srcsrv",
+                ],
+                cwd=os.path.dirname(stream_output_path),
+            )
+            # clean up all the .stream files when done
+            os.remove(stream_output_path)
+        return result
+
+
+class Dumper_Linux(Dumper):
+    objcopy = os.environ["OBJCOPY"] if "OBJCOPY" in os.environ else "objcopy"
+
+    def ShouldProcess(self, file):
+        """This function will allow processing of files that are
+        executable, or end with the .so extension, and additionally
+        file(1) reports as being ELF files.  It expects to find the file
+        command in PATH."""
+        if file.endswith(".so") or os.access(file, os.X_OK):
+            return executables.get_type(file) == executables.ELF
+        return False
+
+    def CopyExeAndDebugInfo(self, file, debug_file, guid, code_file, code_id):
+        # We want to strip out the debug info, and add a
+        # .gnu_debuglink section to the object, so the debugger can
+        # actually load our debug info later.
+        # In some odd cases, the object might already have an irrelevant
+        # .gnu_debuglink section, and objcopy doesn't want to add one in
+        # such cases, so we make it remove it any existing one first.
+        file_dbg = file + ".dbg"
+        if (
+            subprocess.call([self.objcopy, "--only-keep-debug", file, file_dbg]) == 0
+            and subprocess.call(
+                [
+                    self.objcopy,
+                    "--remove-section",
+                    ".gnu_debuglink",
+                    "--add-gnu-debuglink=%s" % file_dbg,
+                    file,
+                ]
+            )
+            == 0
+        ):
+            rel_path = os.path.join(debug_file, guid, debug_file + ".dbg")
+            full_path = os.path.normpath(os.path.join(self.symbol_path, rel_path))
+            shutil.move(file_dbg, full_path)
+            print(rel_path)
+        else:
+            if os.path.isfile(file_dbg):
+                os.unlink(file_dbg)
+
+
+class Dumper_Solaris(Dumper):
+    def ShouldProcess(self, file):
+        """This function will allow processing of files that are
+        executable, or end with the .so extension, and additionally
+        file(1) reports as being ELF files.  It expects to find the file
+        command in PATH."""
+        if file.endswith(".so") or os.access(file, os.X_OK):
+            return executables.get_type(file) == executables.ELF
+        return False
+
+
+class Dumper_Mac(Dumper):
+    def ShouldProcess(self, file):
+        """This function will allow processing of files that are
+        executable, or end with the .dylib extension, and additionally
+        file(1) reports as being Mach-O files.  It expects to find the file
+        command in PATH."""
+        if file.endswith(".dylib") or os.access(file, os.X_OK):
+            return executables.get_type(file) == executables.MACHO
+        return False
+
+    def ProcessFile(self, file, count_ctors=False):
+        print("Starting Mac pre-processing on file: %s" % file, file=sys.stderr)
+        dsymbundle = self.GenerateDSYM(file)
+        if dsymbundle:
+            # kick off new jobs per-arch with our new list of files
+            Dumper.ProcessFile(
+                self, file, dsymbundle=dsymbundle, count_ctors=count_ctors
+            )
+
+    def dump_syms_cmdline(self, file, arch, dsymbundle=None):
+        """
+        Get the commandline used to invoke dump_syms.
+        """
+        # dump_syms wants the path to the original binary and the .dSYM
+        # in order to dump all the symbols.
+        if dsymbundle:
+            # This is the .dSYM bundle.
+            return (
+                [self.dump_syms]
+                + arch.split()
+                + ["--inlines", "-j", "2", dsymbundle, file]
+            )
+        return Dumper.dump_syms_cmdline(self, file, arch)
+
+    def GenerateDSYM(self, file):
+        """dump_syms on Mac needs to be run on a dSYM bundle produced
+        by dsymutil(1), so run dsymutil here and pass the bundle name
+        down to the superclass method instead."""
+        t_start = time.time()
+        print("Running Mac pre-processing on file: %s" % (file,), file=sys.stderr)
+
+        dsymbundle = file + ".dSYM"
+        if os.path.exists(dsymbundle):
+            shutil.rmtree(dsymbundle)
+        dsymutil = buildconfig.substs["DSYMUTIL"]
+        # dsymutil takes --arch=foo instead of -a foo like everything else
+        cmd = (
+            [dsymutil] + [a.replace("-a ", "--arch=") for a in self.archs if a] + [file]
+        )
+        print(" ".join(cmd), file=sys.stderr)
+
+        dsymutil_proc = subprocess.Popen(
+            cmd, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        )
+        dsymout, dsymerr = dsymutil_proc.communicate()
+        if dsymutil_proc.returncode != 0:
+            raise RuntimeError("Error running dsymutil: %s" % dsymerr)
+
+        # Regular dsymutil won't produce a .dSYM for files without symbols.
+        if not os.path.exists(dsymbundle):
+            print("No symbols found in file: %s" % (file,), file=sys.stderr)
+            return False
+
+        # llvm-dsymutil will produce a .dSYM for files without symbols or
+        # debug information, but only sometimes will it warn you about this.
+        # We don't want to run dump_syms on such bundles, because asserts
+        # will fire in debug mode and who knows what will happen in release.
+        #
+        # So we check for the error message and bail if it appears.  If it
+        # doesn't, we carefully check the bundled DWARF to see if dump_syms
+        # will be OK with it.
+        if "warning: no debug symbols in" in dsymerr:
+            print(dsymerr, file=sys.stderr)
+            return False
+
+        contents_dir = os.path.join(dsymbundle, "Contents", "Resources", "DWARF")
+        if not os.path.exists(contents_dir):
+            print(
+                "No DWARF information in .dSYM bundle %s" % (dsymbundle,),
+                file=sys.stderr,
+            )
+            return False
+
+        files = os.listdir(contents_dir)
+        if len(files) != 1:
+            print("Unexpected files in .dSYM bundle %s" % (files,), file=sys.stderr)
+            return False
+
+        otool_out = subprocess.check_output(
+            [buildconfig.substs["OTOOL"], "-l", os.path.join(contents_dir, files[0])],
+            universal_newlines=True,
+        )
+        if "sectname __debug_info" not in otool_out:
+            print("No symbols in .dSYM bundle %s" % (dsymbundle,), file=sys.stderr)
+            return False
+
+        elapsed = time.time() - t_start
+        print("Finished processing %s in %.2fs" % (file, elapsed), file=sys.stderr)
+        return dsymbundle
+
+    def CopyExeAndDebugInfo(self, file, debug_file, guid, code_file, code_id):
+        """ProcessFile has already produced a dSYM bundle, so we should just
+        copy that to the destination directory. However, we'll package it
+        into a .tar because it's a bundle, so it's a directory. |file| here is
+        the original filename."""
+        dsymbundle = file + ".dSYM"
+        rel_path = os.path.join(debug_file, guid, os.path.basename(dsymbundle) + ".tar")
+        full_path = os.path.abspath(os.path.join(self.symbol_path, rel_path))
+        success = subprocess.call(
+            ["tar", "cf", full_path, os.path.basename(dsymbundle)],
+            cwd=os.path.dirname(dsymbundle),
+            stdout=open(os.devnull, "w"),
+            stderr=subprocess.STDOUT,
+        )
+        if success == 0 and os.path.exists(full_path):
+            print(rel_path)
+
+
+# Entry point if called as a standalone program
+
+
+def main():
+    parser = OptionParser(
+        usage="usage: %prog [options] <dump_syms binary> <symbol store path> <debug info files>"
+    )
+    parser.add_option(
+        "-c",
+        "--copy",
+        action="store_true",
+        dest="copy_debug",
+        default=False,
+        help="Copy debug info files into the same directory structure as symbol files",
+    )
+    parser.add_option(
+        "-a",
+        "--archs",
+        action="store",
+        dest="archs",
+        help="Run dump_syms -a <arch> for each space separated"
+        + "cpu architecture in ARCHS (only on OS X)",
+    )
+    parser.add_option(
+        "-s",
+        "--srcdir",
+        action="append",
+        dest="srcdir",
+        default=[],
+        help="Use SRCDIR to determine relative paths to source files",
+    )
+    parser.add_option(
+        "-v",
+        "--vcs-info",
+        action="store_true",
+        dest="vcsinfo",
+        help="Try to retrieve VCS info for each FILE listed in the output",
+    )
+    parser.add_option(
+        "-i",
+        "--source-index",
+        action="store_true",
+        dest="srcsrv",
+        default=False,
+        help="Add source index information to debug files, making them suitable"
+        + " for use in a source server.",
+    )
+    parser.add_option(
+        "--install-manifest",
+        action="append",
+        dest="install_manifests",
+        default=[],
+        help="""Use this install manifest to map filenames back
+to canonical locations in the source repository. Specify
+<install manifest filename>,<install destination> as a comma-separated pair.""",
+    )
+    parser.add_option(
+        "--count-ctors",
+        action="store_true",
+        dest="count_ctors",
+        default=False,
+        help="Count static initializers",
+    )
+    (options, args) = parser.parse_args()
+
+    # check to see if the pdbstr.exe exists
+    if options.srcsrv:
+        if "PDBSTR" not in buildconfig.substs:
+            print("pdbstr was not found by configure.\n", file=sys.stderr)
+            sys.exit(1)
+
+    if len(args) < 3:
+        parser.error("not enough arguments")
+        exit(1)
+
+    try:
+        manifests = validate_install_manifests(options.install_manifests)
+    except (IOError, ValueError) as e:
+        parser.error(str(e))
+        exit(1)
+    file_mapping = make_file_mapping(manifests)
+    _, bucket = get_s3_region_and_bucket()
+    dumper = GetPlatformSpecificDumper(
+        dump_syms=args[0],
+        symbol_path=args[1],
+        copy_debug=options.copy_debug,
+        archs=options.archs,
+        srcdirs=options.srcdir,
+        vcsinfo=options.vcsinfo,
+        srcsrv=options.srcsrv,
+        s3_bucket=bucket,
+        file_mapping=file_mapping,
+    )
+
+    dumper.Process(args[2], options.count_ctors)
+
+
+# run main if run directly
+if __name__ == "__main__":
+    main()