diff options
Diffstat (limited to 'toolkit/crashreporter/google-breakpad/src/tools/mac/upload_system_symbols/upload_system_symbols.go')
-rw-r--r-- | toolkit/crashreporter/google-breakpad/src/tools/mac/upload_system_symbols/upload_system_symbols.go | 432 |
1 files changed, 432 insertions, 0 deletions
diff --git a/toolkit/crashreporter/google-breakpad/src/tools/mac/upload_system_symbols/upload_system_symbols.go b/toolkit/crashreporter/google-breakpad/src/tools/mac/upload_system_symbols/upload_system_symbols.go new file mode 100644 index 0000000000..05a7764abf --- /dev/null +++ b/toolkit/crashreporter/google-breakpad/src/tools/mac/upload_system_symbols/upload_system_symbols.go @@ -0,0 +1,432 @@ +/* Copyright 2014, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* +Tool upload_system_symbols generates and uploads Breakpad symbol files for OS X system libraries. + +This tool shells out to the dump_syms and symupload Breakpad tools. In its default mode, this +will find all dynamic libraries on the system, run dump_syms to create the Breakpad symbol files, +and then upload them to Google's crash infrastructure. + +The tool can also be used to only dump libraries or upload from a directory. See -help for more +information. + +Both i386 and x86_64 architectures will be dumped and uploaded. +*/ +package main + +import ( + "debug/macho" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "path" + "regexp" + "strings" + "sync" + "time" +) + +var ( + breakpadTools = flag.String("breakpad-tools", "out/Release/", "Path to the Breakpad tools directory, containing dump_syms and symupload.") + uploadOnlyPath = flag.String("upload-from", "", "Upload a directory of symbol files that has been dumped independently.") + dumpOnlyPath = flag.String("dump-to", "", "Dump the symbols to the specified directory, but do not upload them.") + systemRoot = flag.String("system-root", "", "Path to the root of the Mac OS X system whose symbols will be dumped.") + dumpArchitecture = flag.String("arch", "", "The CPU architecture for which symbols should be dumped. If not specified, dumps all architectures.") +) + +var ( + // pathsToScan are the subpaths in the systemRoot that should be scanned for shared libraries. + pathsToScan = []string{ + "/System/Library/Components", + "/System/Library/Frameworks", + "/System/Library/PrivateFrameworks", + "/usr/lib", + } + + // optionalPathsToScan is just like pathsToScan, but the paths are permitted to be absent. + optionalPathsToScan = []string{ + // Gone in 10.15. + "/Library/QuickTime", + } + + // uploadServers are the list of servers to which symbols should be uploaded. + uploadServers = []string{ + "https://clients2.google.com/cr/symbol", + "https://clients2.google.com/cr/staging_symbol", + } + + // blacklistRegexps match paths that should be excluded from dumping. + blacklistRegexps = []*regexp.Regexp{ + regexp.MustCompile(`/System/Library/Frameworks/Python\.framework/`), + regexp.MustCompile(`/System/Library/Frameworks/Ruby\.framework/`), + regexp.MustCompile(`_profile\.dylib$`), + regexp.MustCompile(`_debug\.dylib$`), + regexp.MustCompile(`\.a$`), + regexp.MustCompile(`\.dat$`), + } +) + +func main() { + flag.Parse() + log.SetFlags(0) + + var uq *UploadQueue + + if *uploadOnlyPath != "" { + // -upload-from specified, so handle that case early. + uq = StartUploadQueue() + uploadFromDirectory(*uploadOnlyPath, uq) + uq.Wait() + return + } + + if *systemRoot == "" { + log.Fatal("Need a -system-root to dump symbols for") + } + + if *dumpOnlyPath != "" { + // -dump-to specified, so make sure that the path is a directory. + if fi, err := os.Stat(*dumpOnlyPath); err != nil { + log.Fatalf("-dump-to location: %v", err) + } else if !fi.IsDir() { + log.Fatal("-dump-to location is not a directory") + } + } + + dumpPath := *dumpOnlyPath + if *dumpOnlyPath == "" { + // If -dump-to was not specified, then run the upload pipeline and create + // a temporary dump output directory. + uq = StartUploadQueue() + + if p, err := ioutil.TempDir("", "upload_system_symbols"); err != nil { + log.Fatalf("Failed to create temporary directory: %v", err) + } else { + dumpPath = p + defer os.RemoveAll(p) + } + } + + dq := StartDumpQueue(*systemRoot, dumpPath, uq) + dq.Wait() + if uq != nil { + uq.Wait() + } +} + +type WorkerPool struct { + wg sync.WaitGroup +} + +// StartWorkerPool will launch numWorkers goroutines all running workerFunc. +// When workerFunc exits, the goroutine will terminate. +func StartWorkerPool(numWorkers int, workerFunc func()) *WorkerPool { + p := new(WorkerPool) + for i := 0; i < numWorkers; i++ { + p.wg.Add(1) + go func() { + workerFunc() + p.wg.Done() + }() + } + return p +} + +// Wait for all the workers in the pool to complete the workerFunc. +func (p *WorkerPool) Wait() { + p.wg.Wait() +} + +type UploadQueue struct { + *WorkerPool + queue chan string +} + +// StartUploadQueue creates a new worker pool and queue, to which paths to +// Breakpad symbol files may be sent for uploading. +func StartUploadQueue() *UploadQueue { + uq := &UploadQueue{ + queue: make(chan string, 10), + } + uq.WorkerPool = StartWorkerPool(5, uq.worker) + return uq +} + +// Upload enqueues the contents of filepath to be uploaded. +func (uq *UploadQueue) Upload(filepath string) { + uq.queue <- filepath +} + +// Done tells the queue that no more files need to be uploaded. This must be +// called before WorkerPool.Wait. +func (uq *UploadQueue) Done() { + close(uq.queue) +} + +func (uq *UploadQueue) worker() { + symUpload := path.Join(*breakpadTools, "symupload") + + for symfile := range uq.queue { + for _, server := range uploadServers { + for i := 0; i < 3; i++ { // Give each upload 3 attempts to succeed. + cmd := exec.Command(symUpload, symfile, server) + if output, err := cmd.Output(); err == nil { + // Success. No retry needed. + fmt.Printf("Uploaded %s to %s\n", symfile, server) + break + } else { + log.Printf("Error running symupload(%s, %s), attempt %d: %v: %s\n", symfile, server, i, err, output) + time.Sleep(1 * time.Second) + } + } + } + } +} + +type DumpQueue struct { + *WorkerPool + dumpPath string + queue chan dumpRequest + uq *UploadQueue +} + +type dumpRequest struct { + path string + arch string +} + +// StartDumpQueue creates a new worker pool to find all the Mach-O libraries in +// root and dump their symbols to dumpPath. If an UploadQueue is passed, the +// path to the symbol file will be enqueued there, too. +func StartDumpQueue(root, dumpPath string, uq *UploadQueue) *DumpQueue { + dq := &DumpQueue{ + dumpPath: dumpPath, + queue: make(chan dumpRequest), + uq: uq, + } + dq.WorkerPool = StartWorkerPool(12, dq.worker) + + findLibsInRoot(root, dq) + + return dq +} + +// DumpSymbols enqueues the filepath to have its symbols dumped in the specified +// architecture. +func (dq *DumpQueue) DumpSymbols(filepath string, arch string) { + dq.queue <- dumpRequest{ + path: filepath, + arch: arch, + } +} + +func (dq *DumpQueue) Wait() { + dq.WorkerPool.Wait() + if dq.uq != nil { + dq.uq.Done() + } +} + +func (dq *DumpQueue) done() { + close(dq.queue) +} + +func (dq *DumpQueue) worker() { + dumpSyms := path.Join(*breakpadTools, "dump_syms") + + for req := range dq.queue { + filebase := path.Join(dq.dumpPath, strings.Replace(req.path, "/", "_", -1)) + symfile := fmt.Sprintf("%s_%s.sym", filebase, req.arch) + f, err := os.Create(symfile) + if err != nil { + log.Fatalf("Error creating symbol file: %v", err) + } + + cmd := exec.Command(dumpSyms, "-a", req.arch, req.path) + cmd.Stdout = f + err = cmd.Run() + f.Close() + + if err != nil { + os.Remove(symfile) + log.Printf("Error running dump_syms(%s, %s): %v\n", req.arch, req.path, err) + } else if dq.uq != nil { + dq.uq.Upload(symfile) + } + } +} + +// uploadFromDirectory handles the upload-only case and merely uploads all files in +// a directory. +func uploadFromDirectory(directory string, uq *UploadQueue) { + d, err := os.Open(directory) + if err != nil { + log.Fatalf("Could not open directory to upload: %v", err) + } + defer d.Close() + + entries, err := d.Readdirnames(0) + if err != nil { + log.Fatalf("Could not read directory: %v", err) + } + + for _, entry := range entries { + uq.Upload(path.Join(directory, entry)) + } + + uq.Done() +} + +// findQueue is an implementation detail of the DumpQueue that finds all the +// Mach-O files and their architectures. +type findQueue struct { + *WorkerPool + queue chan string + dq *DumpQueue +} + +// findLibsInRoot looks in all the pathsToScan in the root and manages the +// interaction between findQueue and DumpQueue. +func findLibsInRoot(root string, dq *DumpQueue) { + fq := &findQueue{ + queue: make(chan string, 10), + dq: dq, + } + fq.WorkerPool = StartWorkerPool(12, fq.worker) + + for _, p := range pathsToScan { + fq.findLibsInPath(path.Join(root, p), true) + } + + for _, p := range optionalPathsToScan { + fq.findLibsInPath(path.Join(root, p), false) + } + + close(fq.queue) + fq.Wait() + dq.done() +} + +// findLibsInPath recursively walks the directory tree, sending file paths to +// test for being Mach-O to the findQueue. +func (fq *findQueue) findLibsInPath(loc string, mustExist bool) { + d, err := os.Open(loc) + if err != nil { + if !mustExist && os.IsNotExist(err) { + return + } + log.Fatalf("Could not open %s: %v", loc, err) + } + defer d.Close() + + for { + fis, err := d.Readdir(100) + if err != nil && err != io.EOF { + log.Fatalf("Error reading directory %s: %v", loc, err) + } + + for _, fi := range fis { + fp := path.Join(loc, fi.Name()) + if fi.IsDir() { + fq.findLibsInPath(fp, true) + continue + } else if fi.Mode()&os.ModeSymlink != 0 { + continue + } + + // Test the blacklist in the worker to not slow down this main loop. + + fq.queue <- fp + } + + if err == io.EOF { + break + } + } +} + +func (fq *findQueue) worker() { + for fp := range fq.queue { + blacklisted := false + for _, re := range blacklistRegexps { + blacklisted = blacklisted || re.MatchString(fp) + } + if blacklisted { + continue + } + + f, err := os.Open(fp) + if err != nil { + log.Printf("%s: %v", fp, err) + continue + } + + fatFile, err := macho.NewFatFile(f) + if err == nil { + // The file is fat, so dump its architectures. + for _, fatArch := range fatFile.Arches { + fq.dumpMachOFile(fp, fatArch.File) + } + fatFile.Close() + } else if err == macho.ErrNotFat { + // The file isn't fat but may still be MachO. + thinFile, err := macho.NewFile(f) + if err != nil { + log.Printf("%s: %v", fp, err) + continue + } + fq.dumpMachOFile(fp, thinFile) + thinFile.Close() + } else { + f.Close() + } + } +} + +func (fq *findQueue) dumpMachOFile(fp string, image *macho.File) { + if image.Type != MachODylib && image.Type != MachOBundle && image.Type != MachODylinker { + return + } + + arch := getArchStringFromHeader(image.FileHeader) + if arch == "" { + // Don't know about this architecture type. + return + } + + if (*dumpArchitecture != "" && *dumpArchitecture == arch) || *dumpArchitecture == "" { + fq.dq.DumpSymbols(fp, arch) + } +} |