diff options
Diffstat (limited to 'src/cmd/compile/internal/base')
| -rw-r--r-- | src/cmd/compile/internal/base/base.go | 271 | ||||
| -rw-r--r-- | src/cmd/compile/internal/base/bootstrap_false.go | 11 | ||||
| -rw-r--r-- | src/cmd/compile/internal/base/bootstrap_true.go | 11 | ||||
| -rw-r--r-- | src/cmd/compile/internal/base/debug.go | 64 | ||||
| -rw-r--r-- | src/cmd/compile/internal/base/flag.go | 511 | ||||
| -rw-r--r-- | src/cmd/compile/internal/base/hashdebug.go | 323 | ||||
| -rw-r--r-- | src/cmd/compile/internal/base/hashdebug_test.go | 164 | ||||
| -rw-r--r-- | src/cmd/compile/internal/base/link.go | 53 | ||||
| -rw-r--r-- | src/cmd/compile/internal/base/mapfile_mmap.go | 45 | ||||
| -rw-r--r-- | src/cmd/compile/internal/base/mapfile_read.go | 21 | ||||
| -rw-r--r-- | src/cmd/compile/internal/base/print.go | 285 | ||||
| -rw-r--r-- | src/cmd/compile/internal/base/timings.go | 237 |
12 files changed, 1996 insertions, 0 deletions
diff --git a/src/cmd/compile/internal/base/base.go b/src/cmd/compile/internal/base/base.go new file mode 100644 index 0000000..521600b --- /dev/null +++ b/src/cmd/compile/internal/base/base.go @@ -0,0 +1,271 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package base + +import ( + "fmt" + "os" + "runtime" + "runtime/debug" + "runtime/metrics" +) + +var atExitFuncs []func() + +func AtExit(f func()) { + atExitFuncs = append(atExitFuncs, f) +} + +func Exit(code int) { + for i := len(atExitFuncs) - 1; i >= 0; i-- { + f := atExitFuncs[i] + atExitFuncs = atExitFuncs[:i] + f() + } + os.Exit(code) +} + +// To enable tracing support (-t flag), set EnableTrace to true. +const EnableTrace = false + +// forEachGC calls fn each GC cycle until it returns false. +func forEachGC(fn func() bool) { + type T [32]byte // large enough to avoid runtime's tiny object allocator + + var finalizer func(*T) + finalizer = func(p *T) { + if fn() { + runtime.SetFinalizer(p, finalizer) + } + } + + finalizer(new(T)) +} + +// AdjustStartingHeap modifies GOGC so that GC should not occur until the heap +// grows to the requested size. This is intended but not promised, though it +// is true-mostly, depending on when the adjustment occurs and on the +// compiler's input and behavior. Once this size is approximately reached +// GOGC is reset to 100; subsequent GCs may reduce the heap below the requested +// size, but this function does not affect that. +// +// -d=gcadjust=1 enables logging of GOGC adjustment events. +// +// NOTE: If you think this code would help startup time in your own +// application and you decide to use it, please benchmark first to see if it +// actually works for you (it may not: the Go compiler is not typical), and +// whatever the outcome, please leave a comment on bug #56546. This code +// uses supported interfaces, but depends more than we like on +// current+observed behavior of the garbage collector, so if many people need +// this feature, we should consider/propose a better way to accomplish it. +func AdjustStartingHeap(requestedHeapGoal uint64) { + logHeapTweaks := Debug.GCAdjust == 1 + mp := runtime.GOMAXPROCS(0) + gcConcurrency := Flag.LowerC + + const ( + goal = "/gc/heap/goal:bytes" + count = "/gc/cycles/total:gc-cycles" + allocs = "/gc/heap/allocs:bytes" + frees = "/gc/heap/frees:bytes" + ) + + sample := []metrics.Sample{{Name: goal}, {Name: count}, {Name: allocs}, {Name: frees}} + const ( + GOAL = 0 + COUNT = 1 + ALLOCS = 2 + FREES = 3 + ) + + // Assumptions and observations of Go's garbage collector, as of Go 1.17-1.20: + + // - the initial heap goal is 4M, by fiat. It is possible for Go to start + // with a heap as small as 512k, so this may change in the future. + + // - except for the first heap goal, heap goal is a function of + // observed-live at the previous GC and current GOGC. After the first + // GC, adjusting GOGC immediately updates GOGC; before the first GC, + // adjusting GOGC does not modify goal (but the change takes effect after + // the first GC). + + // - the before/after first GC behavior is not guaranteed anywhere, it's + // just behavior, and it's a bad idea to rely on it. + + // - we don't know exactly when GC will run, even after we adjust GOGC; the + // first GC may not have happened yet, may have already happened, or may + // be currently in progress, and GCs can start for several reasons. + + // - forEachGC above will run the provided function at some delay after each + // GC's mark phase terminates; finalizers are run after marking as the + // spans containing finalizable objects are swept, driven by GC + // background activity and allocation demand. + + // - "live at last GC" is not available through the current metrics + // interface. Instead, live is estimated by knowing the adjusted value of + // GOGC and the new heap goal following a GC (this requires knowing that + // at least one GC has occurred): + // estLive = 100 * newGoal / (100 + currentGogc)] + // this new value of GOGC + // newGogc = 100*requestedHeapGoal/estLive - 100 + // will result in the desired goal. The logging code checks that the + // resulting goal is correct. + + // There's a small risk that the finalizer will be slow to run after a GC + // that expands the goal to a huge value, and that this will lead to + // out-of-memory. This doesn't seem to happen; in experiments on a variety + // of machines with a variety of extra loads to disrupt scheduling, the + // worst overshoot observed was 50% past requestedHeapGoal. + + metrics.Read(sample) + for _, s := range sample { + if s.Value.Kind() == metrics.KindBad { + // Just return, a slightly slower compilation is a tolerable outcome. + if logHeapTweaks { + fmt.Fprintf(os.Stderr, "GCAdjust: Regret unexpected KindBad for metric %s\n", s.Name) + } + return + } + } + + // Tinker with GOGC to make the heap grow rapidly at first. + currentGoal := sample[GOAL].Value.Uint64() // Believe this will be 4MByte or less, perhaps 512k + myGogc := 100 * requestedHeapGoal / currentGoal + if myGogc <= 150 { + return + } + + if logHeapTweaks { + sample := append([]metrics.Sample(nil), sample...) // avoid races with GC callback + AtExit(func() { + metrics.Read(sample) + goal := sample[GOAL].Value.Uint64() + count := sample[COUNT].Value.Uint64() + oldGogc := debug.SetGCPercent(100) + if oldGogc == 100 { + fmt.Fprintf(os.Stderr, "GCAdjust: AtExit goal %d gogc %d count %d maxprocs %d gcConcurrency %d\n", + goal, oldGogc, count, mp, gcConcurrency) + } else { + inUse := sample[ALLOCS].Value.Uint64() - sample[FREES].Value.Uint64() + overPct := 100 * (int(inUse) - int(requestedHeapGoal)) / int(requestedHeapGoal) + fmt.Fprintf(os.Stderr, "GCAdjust: AtExit goal %d gogc %d count %d maxprocs %d gcConcurrency %d overPct %d\n", + goal, oldGogc, count, mp, gcConcurrency, overPct) + + } + }) + } + + debug.SetGCPercent(int(myGogc)) + + adjustFunc := func() bool { + + metrics.Read(sample) + goal := sample[GOAL].Value.Uint64() + count := sample[COUNT].Value.Uint64() + + if goal <= requestedHeapGoal { // Stay the course + if logHeapTweaks { + fmt.Fprintf(os.Stderr, "GCAdjust: Reuse GOGC adjust, current goal %d, count is %d, current gogc %d\n", + goal, count, myGogc) + } + return true + } + + // Believe goal has been adjusted upwards, else it would be less-than-or-equal than requestedHeapGoal + calcLive := 100 * goal / (100 + myGogc) + + if 2*calcLive < requestedHeapGoal { // calcLive can exceed requestedHeapGoal! + myGogc = 100*requestedHeapGoal/calcLive - 100 + + if myGogc > 125 { + // Not done growing the heap. + oldGogc := debug.SetGCPercent(int(myGogc)) + + if logHeapTweaks { + // Check that the new goal looks right + inUse := sample[ALLOCS].Value.Uint64() - sample[FREES].Value.Uint64() + metrics.Read(sample) + newGoal := sample[GOAL].Value.Uint64() + pctOff := 100 * (int64(newGoal) - int64(requestedHeapGoal)) / int64(requestedHeapGoal) + // Check that the new goal is close to requested. 3% of make.bash fails this test. Why, TBD. + if pctOff < 2 { + fmt.Fprintf(os.Stderr, "GCAdjust: Retry GOGC adjust, current goal %d, count is %d, gogc was %d, is now %d, calcLive %d pctOff %d\n", + goal, count, oldGogc, myGogc, calcLive, pctOff) + } else { + // The GC is being annoying and not giving us the goal that we requested, say more to help understand when/why. + fmt.Fprintf(os.Stderr, "GCAdjust: Retry GOGC adjust, current goal %d, count is %d, gogc was %d, is now %d, calcLive %d pctOff %d inUse %d\n", + goal, count, oldGogc, myGogc, calcLive, pctOff, inUse) + } + } + return true + } + } + + // In this case we're done boosting GOGC, set it to 100 and don't set a new finalizer. + oldGogc := debug.SetGCPercent(100) + // inUse helps estimate how late the finalizer ran; at the instant the previous GC ended, + // it was (in theory) equal to the previous GC's heap goal. In a growing heap it is + // expected to grow to the new heap goal. + inUse := sample[ALLOCS].Value.Uint64() - sample[FREES].Value.Uint64() + overPct := 100 * (int(inUse) - int(requestedHeapGoal)) / int(requestedHeapGoal) + if logHeapTweaks { + fmt.Fprintf(os.Stderr, "GCAdjust: Reset GOGC adjust, old goal %d, count is %d, gogc was %d, calcLive %d inUse %d overPct %d\n", + goal, count, oldGogc, calcLive, inUse, overPct) + } + return false + } + + forEachGC(adjustFunc) +} + +func Compiling(pkgs []string) bool { + if Ctxt.Pkgpath != "" { + for _, p := range pkgs { + if Ctxt.Pkgpath == p { + return true + } + } + } + + return false +} + +// The racewalk pass is currently handled in three parts. +// +// First, for flag_race, it inserts calls to racefuncenter and +// racefuncexit at the start and end (respectively) of each +// function. This is handled below. +// +// Second, during buildssa, it inserts appropriate instrumentation +// calls immediately before each memory load or store. This is handled +// by the (*state).instrument method in ssa.go, so here we just set +// the Func.InstrumentBody flag as needed. For background on why this +// is done during SSA construction rather than a separate SSA pass, +// see issue #19054. +// +// Third we remove calls to racefuncenter and racefuncexit, for leaf +// functions without instrumented operations. This is done as part of +// ssa opt pass via special rule. + +// TODO(dvyukov): do not instrument initialization as writes: +// a := make([]int, 10) + +// Do not instrument the following packages at all, +// at best instrumentation would cause infinite recursion. +var NoInstrumentPkgs = []string{ + "runtime/internal/atomic", + "runtime/internal/math", + "runtime/internal/sys", + "runtime/internal/syscall", + "runtime", + "runtime/race", + "runtime/msan", + "runtime/asan", + "internal/cpu", +} + +// Don't insert racefuncenter/racefuncexit into the following packages. +// Memory accesses in the packages are either uninteresting or will cause false positives. +var NoRacePkgs = []string{"sync", "sync/atomic"} diff --git a/src/cmd/compile/internal/base/bootstrap_false.go b/src/cmd/compile/internal/base/bootstrap_false.go new file mode 100644 index 0000000..ea6da43 --- /dev/null +++ b/src/cmd/compile/internal/base/bootstrap_false.go @@ -0,0 +1,11 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !compiler_bootstrap + +package base + +// CompilerBootstrap reports whether the current compiler binary was +// built with -tags=compiler_bootstrap. +const CompilerBootstrap = false diff --git a/src/cmd/compile/internal/base/bootstrap_true.go b/src/cmd/compile/internal/base/bootstrap_true.go new file mode 100644 index 0000000..d0c6c88 --- /dev/null +++ b/src/cmd/compile/internal/base/bootstrap_true.go @@ -0,0 +1,11 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build compiler_bootstrap + +package base + +// CompilerBootstrap reports whether the current compiler binary was +// built with -tags=compiler_bootstrap. +const CompilerBootstrap = true diff --git a/src/cmd/compile/internal/base/debug.go b/src/cmd/compile/internal/base/debug.go new file mode 100644 index 0000000..ee42696 --- /dev/null +++ b/src/cmd/compile/internal/base/debug.go @@ -0,0 +1,64 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Debug arguments, set by -d flag. + +package base + +// Debug holds the parsed debugging configuration values. +var Debug DebugFlags + +// DebugFlags defines the debugging configuration values (see var Debug). +// Each struct field is a different value, named for the lower-case of the field name. +// Each field must be an int or string and must have a `help` struct tag. +// +// The -d option takes a comma-separated list of settings. +// Each setting is name=value; for ints, name is short for name=1. +type DebugFlags struct { + Append int `help:"print information about append compilation"` + Checkptr int `help:"instrument unsafe pointer conversions\n0: instrumentation disabled\n1: conversions involving unsafe.Pointer are instrumented\n2: conversions to unsafe.Pointer force heap allocation" concurrent:"ok"` + Closure int `help:"print information about closure compilation"` + DclStack int `help:"run internal dclstack check"` + Defer int `help:"print information about defer compilation"` + DisableNil int `help:"disable nil checks" concurrent:"ok"` + DumpPtrs int `help:"show Node pointers values in dump output"` + DwarfInl int `help:"print information about DWARF inlined function creation"` + Export int `help:"print export data"` + Fmahash string `help:"hash value for use in debugging platform-dependent multiply-add use" concurrent:"ok"` + GCAdjust int `help:"log adjustments to GOGC" concurrent:"ok"` + GCCheck int `help:"check heap/gc use by compiler" concurrent:"ok"` + GCProg int `help:"print dump of GC programs"` + Gossahash string `help:"hash value for use in debugging the compiler"` + InlFuncsWithClosures int `help:"allow functions with closures to be inlined" concurrent:"ok"` + InlStaticInit int `help:"allow static initialization of inlined calls" concurrent:"ok"` + InterfaceCycles int `help:"allow anonymous interface cycles"` + Libfuzzer int `help:"enable coverage instrumentation for libfuzzer"` + LocationLists int `help:"print information about DWARF location list creation"` + Nil int `help:"print information about nil checks"` + NoOpenDefer int `help:"disable open-coded defers" concurrent:"ok"` + NoRefName int `help:"do not include referenced symbol names in object file" concurrent:"ok"` + PCTab string `help:"print named pc-value table\nOne of: pctospadj, pctofile, pctoline, pctoinline, pctopcdata"` + Panic int `help:"show all compiler panics"` + Reshape int `help:"print information about expression reshaping"` + Shapify int `help:"print information about shaping recursive types"` + Slice int `help:"print information about slice compilation"` + SoftFloat int `help:"force compiler to emit soft-float code" concurrent:"ok"` + SyncFrames int `help:"how many writer stack frames to include at sync points in unified export data"` + TypeAssert int `help:"print information about type assertion inlining"` + TypecheckInl int `help:"eager typechecking of inline function bodies" concurrent:"ok"` + Unified int `help:"enable unified IR construction"` + WB int `help:"print information about write barriers"` + ABIWrap int `help:"print information about ABI wrapper generation"` + MayMoreStack string `help:"call named function before all stack growth checks" concurrent:"ok"` + PGOInlineCDFThreshold string `help:"cummulative threshold percentage for determining call sites as hot candidates for inlining" concurrent:"ok"` + PGOInlineBudget int `help:"inline budget for hot functions" concurrent:"ok"` + PGOInline int `help:"debug profile-guided inlining"` + + ConcurrentOk bool // true if only concurrentOk flags seen +} + +// DebugSSA is called to set a -d ssa/... option. +// If nil, those options are reported as invalid options. +// If DebugSSA returns a non-empty string, that text is reported as a compiler error. +var DebugSSA func(phase, flag string, val int, valString string) string diff --git a/src/cmd/compile/internal/base/flag.go b/src/cmd/compile/internal/base/flag.go new file mode 100644 index 0000000..be555c3 --- /dev/null +++ b/src/cmd/compile/internal/base/flag.go @@ -0,0 +1,511 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package base + +import ( + "encoding/json" + "flag" + "fmt" + "internal/buildcfg" + "internal/coverage" + "internal/platform" + "log" + "os" + "reflect" + "runtime" + "strings" + + "cmd/internal/obj" + "cmd/internal/objabi" + "cmd/internal/sys" +) + +func usage() { + fmt.Fprintf(os.Stderr, "usage: compile [options] file.go...\n") + objabi.Flagprint(os.Stderr) + Exit(2) +} + +// Flag holds the parsed command-line flags. +// See ParseFlag for non-zero defaults. +var Flag CmdFlags + +// A CountFlag is a counting integer flag. +// It accepts -name=value to set the value directly, +// but it also accepts -name with no =value to increment the count. +type CountFlag int + +// CmdFlags defines the command-line flags (see var Flag). +// Each struct field is a different flag, by default named for the lower-case of the field name. +// If the flag name is a single letter, the default flag name is left upper-case. +// If the flag name is "Lower" followed by a single letter, the default flag name is the lower-case of the last letter. +// +// If this default flag name can't be made right, the `flag` struct tag can be used to replace it, +// but this should be done only in exceptional circumstances: it helps everyone if the flag name +// is obvious from the field name when the flag is used elsewhere in the compiler sources. +// The `flag:"-"` struct tag makes a field invisible to the flag logic and should also be used sparingly. +// +// Each field must have a `help` struct tag giving the flag help message. +// +// The allowed field types are bool, int, string, pointers to those (for values stored elsewhere), +// CountFlag (for a counting flag), and func(string) (for a flag that uses special code for parsing). +type CmdFlags struct { + // Single letters + B CountFlag "help:\"disable bounds checking\"" + C CountFlag "help:\"disable printing of columns in error messages\"" + D string "help:\"set relative `path` for local imports\"" + E CountFlag "help:\"debug symbol export\"" + I func(string) "help:\"add `directory` to import search path\"" + K CountFlag "help:\"debug missing line numbers\"" + L CountFlag "help:\"also show actual source file names in error messages for positions affected by //line directives\"" + N CountFlag "help:\"disable optimizations\"" + S CountFlag "help:\"print assembly listing\"" + // V is added by objabi.AddVersionFlag + W CountFlag "help:\"debug parse tree after type checking\"" + + LowerC int "help:\"concurrency during compilation (1 means no concurrency)\"" + LowerD flag.Value "help:\"enable debugging settings; try -d help\"" + LowerE CountFlag "help:\"no limit on number of errors reported\"" + LowerH CountFlag "help:\"halt on error\"" + LowerJ CountFlag "help:\"debug runtime-initialized variables\"" + LowerL CountFlag "help:\"disable inlining\"" + LowerM CountFlag "help:\"print optimization decisions\"" + LowerO string "help:\"write output to `file`\"" + LowerP *string "help:\"set expected package import `path`\"" // &Ctxt.Pkgpath, set below + LowerR CountFlag "help:\"debug generated wrappers\"" + LowerT bool "help:\"enable tracing for debugging the compiler\"" + LowerW CountFlag "help:\"debug type checking\"" + LowerV *bool "help:\"increase debug verbosity\"" + + // Special characters + Percent CountFlag "flag:\"%\" help:\"debug non-static initializers\"" + CompilingRuntime bool "flag:\"+\" help:\"compiling runtime\"" + + // Longer names + AsmHdr string "help:\"write assembly header to `file`\"" + ASan bool "help:\"build code compatible with C/C++ address sanitizer\"" + Bench string "help:\"append benchmark times to `file`\"" + BlockProfile string "help:\"write block profile to `file`\"" + BuildID string "help:\"record `id` as the build id in the export metadata\"" + CPUProfile string "help:\"write cpu profile to `file`\"" + Complete bool "help:\"compiling complete package (no C or assembly)\"" + ClobberDead bool "help:\"clobber dead stack slots (for debugging)\"" + ClobberDeadReg bool "help:\"clobber dead registers (for debugging)\"" + Dwarf bool "help:\"generate DWARF symbols\"" + DwarfBASEntries *bool "help:\"use base address selection entries in DWARF\"" // &Ctxt.UseBASEntries, set below + DwarfLocationLists *bool "help:\"add location lists to DWARF in optimized mode\"" // &Ctxt.Flag_locationlists, set below + Dynlink *bool "help:\"support references to Go symbols defined in other shared libraries\"" // &Ctxt.Flag_dynlink, set below + EmbedCfg func(string) "help:\"read go:embed configuration from `file`\"" + GenDwarfInl int "help:\"generate DWARF inline info records\"" // 0=disabled, 1=funcs, 2=funcs+formals/locals + GoVersion string "help:\"required version of the runtime\"" + ImportCfg func(string) "help:\"read import configuration from `file`\"" + InstallSuffix string "help:\"set pkg directory `suffix`\"" + JSON string "help:\"version,file for JSON compiler/optimizer detail output\"" + Lang string "help:\"Go language version source code expects\"" + LinkObj string "help:\"write linker-specific object to `file`\"" + LinkShared *bool "help:\"generate code that will be linked against Go shared libraries\"" // &Ctxt.Flag_linkshared, set below + Live CountFlag "help:\"debug liveness analysis\"" + MSan bool "help:\"build code compatible with C/C++ memory sanitizer\"" + MemProfile string "help:\"write memory profile to `file`\"" + MemProfileRate int "help:\"set runtime.MemProfileRate to `rate`\"" + MutexProfile string "help:\"write mutex profile to `file`\"" + NoLocalImports bool "help:\"reject local (relative) imports\"" + CoverageCfg func(string) "help:\"read coverage configuration from `file`\"" + Pack bool "help:\"write to file.a instead of file.o\"" + Race bool "help:\"enable race detector\"" + Shared *bool "help:\"generate code that can be linked into a shared library\"" // &Ctxt.Flag_shared, set below + SmallFrames bool "help:\"reduce the size limit for stack allocated objects\"" // small stacks, to diagnose GC latency; see golang.org/issue/27732 + Spectre string "help:\"enable spectre mitigations in `list` (all, index, ret)\"" + Std bool "help:\"compiling standard library\"" + SymABIs string "help:\"read symbol ABIs from `file`\"" + TraceProfile string "help:\"write an execution trace to `file`\"" + TrimPath string "help:\"remove `prefix` from recorded source file paths\"" + WB bool "help:\"enable write barrier\"" // TODO: remove + OldComparable bool "help:\"enable old comparable semantics\"" // TODO: remove for Go 1.21 + PgoProfile string "help:\"read profile from `file`\"" + + // Configuration derived from flags; not a flag itself. + Cfg struct { + Embed struct { // set by -embedcfg + Patterns map[string][]string + Files map[string]string + } + ImportDirs []string // appended to by -I + ImportMap map[string]string // set by -importcfg + PackageFile map[string]string // set by -importcfg; nil means not in use + CoverageInfo *coverage.CoverFixupConfig // set by -coveragecfg + SpectreIndex bool // set by -spectre=index or -spectre=all + // Whether we are adding any sort of code instrumentation, such as + // when the race detector is enabled. + Instrumenting bool + } +} + +// ParseFlags parses the command-line flags into Flag. +func ParseFlags() { + Flag.I = addImportDir + + Flag.LowerC = runtime.GOMAXPROCS(0) + Flag.LowerD = objabi.NewDebugFlag(&Debug, DebugSSA) + Flag.LowerP = &Ctxt.Pkgpath + Flag.LowerV = &Ctxt.Debugvlog + + Flag.Dwarf = buildcfg.GOARCH != "wasm" + Flag.DwarfBASEntries = &Ctxt.UseBASEntries + Flag.DwarfLocationLists = &Ctxt.Flag_locationlists + *Flag.DwarfLocationLists = true + Flag.Dynlink = &Ctxt.Flag_dynlink + Flag.EmbedCfg = readEmbedCfg + Flag.GenDwarfInl = 2 + Flag.ImportCfg = readImportCfg + Flag.CoverageCfg = readCoverageCfg + Flag.LinkShared = &Ctxt.Flag_linkshared + Flag.Shared = &Ctxt.Flag_shared + Flag.WB = true + + Debug.ConcurrentOk = true + Debug.InlFuncsWithClosures = 1 + Debug.InlStaticInit = 0 + if buildcfg.Experiment.Unified { + Debug.Unified = 1 + } + Debug.SyncFrames = -1 // disable sync markers by default + + Debug.Checkptr = -1 // so we can tell whether it is set explicitly + + Flag.Cfg.ImportMap = make(map[string]string) + + objabi.AddVersionFlag() // -V + registerFlags() + objabi.Flagparse(usage) + + if gcd := os.Getenv("GOCOMPILEDEBUG"); gcd != "" { + // This will only override the flags set in gcd; + // any others set on the command line remain set. + Flag.LowerD.Set(gcd) + } + + if Debug.Gossahash != "" { + hashDebug = NewHashDebug("gosshash", Debug.Gossahash, nil) + } + + if Debug.Fmahash != "" { + FmaHash = NewHashDebug("fmahash", Debug.Fmahash, nil) + } + + if Flag.MSan && !platform.MSanSupported(buildcfg.GOOS, buildcfg.GOARCH) { + log.Fatalf("%s/%s does not support -msan", buildcfg.GOOS, buildcfg.GOARCH) + } + if Flag.ASan && !platform.ASanSupported(buildcfg.GOOS, buildcfg.GOARCH) { + log.Fatalf("%s/%s does not support -asan", buildcfg.GOOS, buildcfg.GOARCH) + } + if Flag.Race && !platform.RaceDetectorSupported(buildcfg.GOOS, buildcfg.GOARCH) { + log.Fatalf("%s/%s does not support -race", buildcfg.GOOS, buildcfg.GOARCH) + } + if (*Flag.Shared || *Flag.Dynlink || *Flag.LinkShared) && !Ctxt.Arch.InFamily(sys.AMD64, sys.ARM, sys.ARM64, sys.I386, sys.PPC64, sys.RISCV64, sys.S390X) { + log.Fatalf("%s/%s does not support -shared", buildcfg.GOOS, buildcfg.GOARCH) + } + parseSpectre(Flag.Spectre) // left as string for RecordFlags + + Ctxt.Flag_shared = Ctxt.Flag_dynlink || Ctxt.Flag_shared + Ctxt.Flag_optimize = Flag.N == 0 + Ctxt.Debugasm = int(Flag.S) + Ctxt.Flag_maymorestack = Debug.MayMoreStack + Ctxt.Flag_noRefName = Debug.NoRefName != 0 + + if flag.NArg() < 1 { + usage() + } + + if Flag.GoVersion != "" && Flag.GoVersion != runtime.Version() { + fmt.Printf("compile: version %q does not match go tool version %q\n", runtime.Version(), Flag.GoVersion) + Exit(2) + } + + if *Flag.LowerP == "" { + *Flag.LowerP = obj.UnlinkablePkg + } + + if Flag.LowerO == "" { + p := flag.Arg(0) + if i := strings.LastIndex(p, "/"); i >= 0 { + p = p[i+1:] + } + if runtime.GOOS == "windows" { + if i := strings.LastIndex(p, `\`); i >= 0 { + p = p[i+1:] + } + } + if i := strings.LastIndex(p, "."); i >= 0 { + p = p[:i] + } + suffix := ".o" + if Flag.Pack { + suffix = ".a" + } + Flag.LowerO = p + suffix + } + switch { + case Flag.Race && Flag.MSan: + log.Fatal("cannot use both -race and -msan") + case Flag.Race && Flag.ASan: + log.Fatal("cannot use both -race and -asan") + case Flag.MSan && Flag.ASan: + log.Fatal("cannot use both -msan and -asan") + } + if Flag.Race || Flag.MSan || Flag.ASan { + // -race, -msan and -asan imply -d=checkptr for now. + if Debug.Checkptr == -1 { // if not set explicitly + Debug.Checkptr = 1 + } + } + + if Flag.CompilingRuntime && Flag.N != 0 { + log.Fatal("cannot disable optimizations while compiling runtime") + } + if Flag.LowerC < 1 { + log.Fatalf("-c must be at least 1, got %d", Flag.LowerC) + } + if !concurrentBackendAllowed() { + Flag.LowerC = 1 + } + + if Flag.CompilingRuntime { + // Runtime can't use -d=checkptr, at least not yet. + Debug.Checkptr = 0 + + // Fuzzing the runtime isn't interesting either. + Debug.Libfuzzer = 0 + } + + if Debug.Checkptr == -1 { // if not set explicitly + Debug.Checkptr = 0 + } + + // set via a -d flag + Ctxt.Debugpcln = Debug.PCTab +} + +// registerFlags adds flag registrations for all the fields in Flag. +// See the comment on type CmdFlags for the rules. +func registerFlags() { + var ( + boolType = reflect.TypeOf(bool(false)) + intType = reflect.TypeOf(int(0)) + stringType = reflect.TypeOf(string("")) + ptrBoolType = reflect.TypeOf(new(bool)) + ptrIntType = reflect.TypeOf(new(int)) + ptrStringType = reflect.TypeOf(new(string)) + countType = reflect.TypeOf(CountFlag(0)) + funcType = reflect.TypeOf((func(string))(nil)) + ) + + v := reflect.ValueOf(&Flag).Elem() + t := v.Type() + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if f.Name == "Cfg" { + continue + } + + var name string + if len(f.Name) == 1 { + name = f.Name + } else if len(f.Name) == 6 && f.Name[:5] == "Lower" && 'A' <= f.Name[5] && f.Name[5] <= 'Z' { + name = string(rune(f.Name[5] + 'a' - 'A')) + } else { + name = strings.ToLower(f.Name) + } + if tag := f.Tag.Get("flag"); tag != "" { + name = tag + } + + help := f.Tag.Get("help") + if help == "" { + panic(fmt.Sprintf("base.Flag.%s is missing help text", f.Name)) + } + + if k := f.Type.Kind(); (k == reflect.Ptr || k == reflect.Func) && v.Field(i).IsNil() { + panic(fmt.Sprintf("base.Flag.%s is uninitialized %v", f.Name, f.Type)) + } + + switch f.Type { + case boolType: + p := v.Field(i).Addr().Interface().(*bool) + flag.BoolVar(p, name, *p, help) + case intType: + p := v.Field(i).Addr().Interface().(*int) + flag.IntVar(p, name, *p, help) + case stringType: + p := v.Field(i).Addr().Interface().(*string) + flag.StringVar(p, name, *p, help) + case ptrBoolType: + p := v.Field(i).Interface().(*bool) + flag.BoolVar(p, name, *p, help) + case ptrIntType: + p := v.Field(i).Interface().(*int) + flag.IntVar(p, name, *p, help) + case ptrStringType: + p := v.Field(i).Interface().(*string) + flag.StringVar(p, name, *p, help) + case countType: + p := (*int)(v.Field(i).Addr().Interface().(*CountFlag)) + objabi.Flagcount(name, help, p) + case funcType: + f := v.Field(i).Interface().(func(string)) + objabi.Flagfn1(name, help, f) + default: + if val, ok := v.Field(i).Interface().(flag.Value); ok { + flag.Var(val, name, help) + } else { + panic(fmt.Sprintf("base.Flag.%s has unexpected type %s", f.Name, f.Type)) + } + } + } +} + +// concurrentFlagOk reports whether the current compiler flags +// are compatible with concurrent compilation. +func concurrentFlagOk() bool { + // TODO(rsc): Many of these are fine. Remove them. + return Flag.Percent == 0 && + Flag.E == 0 && + Flag.K == 0 && + Flag.L == 0 && + Flag.LowerH == 0 && + Flag.LowerJ == 0 && + Flag.LowerM == 0 && + Flag.LowerR == 0 +} + +func concurrentBackendAllowed() bool { + if !concurrentFlagOk() { + return false + } + + // Debug.S by itself is ok, because all printing occurs + // while writing the object file, and that is non-concurrent. + // Adding Debug_vlog, however, causes Debug.S to also print + // while flushing the plist, which happens concurrently. + if Ctxt.Debugvlog || !Debug.ConcurrentOk || Flag.Live > 0 { + return false + } + // TODO: Test and delete this condition. + if buildcfg.Experiment.FieldTrack { + return false + } + // TODO: fix races and enable the following flags + if Ctxt.Flag_dynlink || Flag.Race { + return false + } + return true +} + +func addImportDir(dir string) { + if dir != "" { + Flag.Cfg.ImportDirs = append(Flag.Cfg.ImportDirs, dir) + } +} + +func readImportCfg(file string) { + if Flag.Cfg.ImportMap == nil { + Flag.Cfg.ImportMap = make(map[string]string) + } + Flag.Cfg.PackageFile = map[string]string{} + data, err := os.ReadFile(file) + if err != nil { + log.Fatalf("-importcfg: %v", err) + } + + for lineNum, line := range strings.Split(string(data), "\n") { + lineNum++ // 1-based + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + var verb, args string + if i := strings.Index(line, " "); i < 0 { + verb = line + } else { + verb, args = line[:i], strings.TrimSpace(line[i+1:]) + } + var before, after string + if i := strings.Index(args, "="); i >= 0 { + before, after = args[:i], args[i+1:] + } + switch verb { + default: + log.Fatalf("%s:%d: unknown directive %q", file, lineNum, verb) + case "importmap": + if before == "" || after == "" { + log.Fatalf(`%s:%d: invalid importmap: syntax is "importmap old=new"`, file, lineNum) + } + Flag.Cfg.ImportMap[before] = after + case "packagefile": + if before == "" || after == "" { + log.Fatalf(`%s:%d: invalid packagefile: syntax is "packagefile path=filename"`, file, lineNum) + } + Flag.Cfg.PackageFile[before] = after + } + } +} + +func readCoverageCfg(file string) { + var cfg coverage.CoverFixupConfig + data, err := os.ReadFile(file) + if err != nil { + log.Fatalf("-coveragecfg: %v", err) + } + if err := json.Unmarshal(data, &cfg); err != nil { + log.Fatalf("error reading -coveragecfg file %q: %v", file, err) + } + Flag.Cfg.CoverageInfo = &cfg +} + +func readEmbedCfg(file string) { + data, err := os.ReadFile(file) + if err != nil { + log.Fatalf("-embedcfg: %v", err) + } + if err := json.Unmarshal(data, &Flag.Cfg.Embed); err != nil { + log.Fatalf("%s: %v", file, err) + } + if Flag.Cfg.Embed.Patterns == nil { + log.Fatalf("%s: invalid embedcfg: missing Patterns", file) + } + if Flag.Cfg.Embed.Files == nil { + log.Fatalf("%s: invalid embedcfg: missing Files", file) + } +} + +// parseSpectre parses the spectre configuration from the string s. +func parseSpectre(s string) { + for _, f := range strings.Split(s, ",") { + f = strings.TrimSpace(f) + switch f { + default: + log.Fatalf("unknown setting -spectre=%s", f) + case "": + // nothing + case "all": + Flag.Cfg.SpectreIndex = true + Ctxt.Retpoline = true + case "index": + Flag.Cfg.SpectreIndex = true + case "ret": + Ctxt.Retpoline = true + } + } + + if Flag.Cfg.SpectreIndex { + switch buildcfg.GOARCH { + case "amd64": + // ok + default: + log.Fatalf("GOARCH=%s does not support -spectre=index", buildcfg.GOARCH) + } + } +} diff --git a/src/cmd/compile/internal/base/hashdebug.go b/src/cmd/compile/internal/base/hashdebug.go new file mode 100644 index 0000000..6c4821b --- /dev/null +++ b/src/cmd/compile/internal/base/hashdebug.go @@ -0,0 +1,323 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package base + +import ( + "bytes" + "cmd/internal/notsha256" + "cmd/internal/obj" + "cmd/internal/src" + "fmt" + "io" + "os" + "strconv" + "strings" + "sync" +) + +type writeSyncer interface { + io.Writer + Sync() error +} + +type hashAndMask struct { + // a hash h matches if (h^hash)&mask == 0 + hash uint64 + mask uint64 + name string // base name, or base name + "0", "1", etc. +} + +type HashDebug struct { + mu sync.Mutex // for logfile, posTmp, bytesTmp + name string // base name of the flag/variable. + // what file (if any) receives the yes/no logging? + // default is os.Stdout + logfile writeSyncer + posTmp []src.Pos + bytesTmp bytes.Buffer + matches []hashAndMask // A hash matches if one of these matches. + yes, no bool +} + +// The default compiler-debugging HashDebug, for "-d=gossahash=..." +var hashDebug *HashDebug +var FmaHash *HashDebug + +// DebugHashMatch reports whether debug variable Gossahash +// +// 1. is empty (returns true; this is a special more-quickly implemented case of 4 below) +// +// 2. is "y" or "Y" (returns true) +// +// 3. is "n" or "N" (returns false) +// +// 4. is a suffix of the sha1 hash of pkgAndName (returns true) +// +// 5. OR +// if the value is in the regular language "[01]+(;[01]+)+" +// test the [01]+ substrings after in order returning true +// for the first one that suffix-matches. The substrings AFTER +// the first semicolon are numbered 0,1, etc and are named +// fmt.Sprintf("%s%d", varname, number) +// Clause 5 is not really intended for human use and only +// matters for failures that require multiple triggers. +// +// Otherwise it returns false. +// +// Unless Flags.Gossahash is empty, when DebugHashMatch returns true the message +// +// "%s triggered %s\n", varname, pkgAndName +// +// is printed on the file named in environment variable GSHS_LOGFILE, +// or standard out if that is empty. "Varname" is either the name of +// the variable or the name of the substring, depending on which matched. +// +// Typical use: +// +// 1. you make a change to the compiler, say, adding a new phase +// +// 2. it is broken in some mystifying way, for example, make.bash builds a broken +// compiler that almost works, but crashes compiling a test in run.bash. +// +// 3. add this guard to the code, which by default leaves it broken, but does not +// run the broken new code if Flags.Gossahash is non-empty and non-matching: +// +// if !base.DebugHashMatch(ir.PkgFuncName(fn)) { +// return nil // early exit, do nothing +// } +// +// 4. rebuild w/o the bad code, +// GOCOMPILEDEBUG=gossahash=n ./all.bash +// to verify that you put the guard in the right place with the right sense of the test. +// +// 5. use github.com/dr2chase/gossahash to search for the error: +// +// go install github.com/dr2chase/gossahash@latest +// +// gossahash -- <the thing that fails> +// +// for example: GOMAXPROCS=1 gossahash -- ./all.bash +// +// 6. gossahash should return a single function whose miscompilation +// causes the problem, and you can focus on that. +func DebugHashMatch(pkgAndName string) bool { + return hashDebug.DebugHashMatch(pkgAndName) +} + +// HasDebugHash returns true if Flags.Gossahash is non-empty, which +// results in hashDebug being not-nil. I.e., if !HasDebugHash(), +// there is no need to create the string for hashing and testing. +func HasDebugHash() bool { + return hashDebug != nil +} + +func toHashAndMask(s, varname string) hashAndMask { + l := len(s) + if l > 64 { + s = s[l-64:] + l = 64 + } + m := ^(^uint64(0) << l) + h, err := strconv.ParseUint(s, 2, 64) + if err != nil { + Fatalf("Could not parse %s (=%s) as a binary number", varname, s) + } + + return hashAndMask{name: varname, hash: h, mask: m} +} + +// NewHashDebug returns a new hash-debug tester for the +// environment variable ev. If ev is not set, it returns +// nil, allowing a lightweight check for normal-case behavior. +func NewHashDebug(ev, s string, file writeSyncer) *HashDebug { + if s == "" { + return nil + } + + hd := &HashDebug{name: ev, logfile: file} + switch s[0] { + case 'y', 'Y': + hd.yes = true + return hd + case 'n', 'N': + hd.no = true + return hd + } + ss := strings.Split(s, "/") + hd.matches = append(hd.matches, toHashAndMask(ss[0], ev)) + // hash searches may use additional EVs with 0, 1, 2, ... suffixes. + for i := 1; i < len(ss); i++ { + evi := fmt.Sprintf("%s%d", ev, i-1) // convention is extras begin indexing at zero + hd.matches = append(hd.matches, toHashAndMask(ss[i], evi)) + } + return hd + +} + +func hashOf(pkgAndName string, param uint64) uint64 { + return hashOfBytes([]byte(pkgAndName), param) +} + +func hashOfBytes(sbytes []byte, param uint64) uint64 { + hbytes := notsha256.Sum256(sbytes) + hash := uint64(hbytes[7])<<56 + uint64(hbytes[6])<<48 + + uint64(hbytes[5])<<40 + uint64(hbytes[4])<<32 + + uint64(hbytes[3])<<24 + uint64(hbytes[2])<<16 + + uint64(hbytes[1])<<8 + uint64(hbytes[0]) + + if param != 0 { + // Because param is probably a line number, probably near zero, + // hash it up a little bit, but even so only the lower-order bits + // likely matter because search focuses on those. + p0 := param + uint64(hbytes[9]) + uint64(hbytes[10])<<8 + + uint64(hbytes[11])<<16 + uint64(hbytes[12])<<24 + + p1 := param + uint64(hbytes[13]) + uint64(hbytes[14])<<8 + + uint64(hbytes[15])<<16 + uint64(hbytes[16])<<24 + + param += p0 * p1 + param ^= param>>17 ^ param<<47 + } + + return hash ^ param +} + +// DebugHashMatch returns true if either the variable used to create d is +// unset, or if its value is y, or if it is a suffix of the base-two +// representation of the hash of pkgAndName. If the variable is not nil, +// then a true result is accompanied by stylized output to d.logfile, which +// is used for automated bug search. +func (d *HashDebug) DebugHashMatch(pkgAndName string) bool { + return d.DebugHashMatchParam(pkgAndName, 0) +} + +// DebugHashMatchParam returns true if either the variable used to create d is +// unset, or if its value is y, or if it is a suffix of the base-two +// representation of the hash of pkgAndName and param. If the variable is not +// nil, then a true result is accompanied by stylized output to d.logfile, +// which is used for automated bug search. +func (d *HashDebug) DebugHashMatchParam(pkgAndName string, param uint64) bool { + if d == nil { + return true + } + if d.no { + return false + } + + if d.yes { + d.logDebugHashMatch(d.name, pkgAndName, "y", param) + return true + } + + hash := hashOf(pkgAndName, param) + + for _, m := range d.matches { + if (m.hash^hash)&m.mask == 0 { + hstr := "" + if hash == 0 { + hstr = "0" + } else { + for ; hash != 0; hash = hash >> 1 { + hstr = string('0'+byte(hash&1)) + hstr + } + } + d.logDebugHashMatch(m.name, pkgAndName, hstr, param) + return true + } + } + return false +} + +// DebugHashMatchPos is similar to DebugHashMatchParam, but for hash computation +// it uses the source position including all inlining information instead of +// package name and path. The output trigger string is prefixed with "POS=" so +// that tools processing the output can reliably tell the difference. The mutex +// locking is also more frequent and more granular. +func (d *HashDebug) DebugHashMatchPos(ctxt *obj.Link, pos src.XPos) bool { + if d == nil { + return true + } + if d.no { + return false + } + d.mu.Lock() + defer d.mu.Unlock() + + b := d.bytesForPos(ctxt, pos) + + if d.yes { + d.logDebugHashMatchLocked(d.name, string(b), "y", 0) + return true + } + + hash := hashOfBytes(b, 0) + + for _, m := range d.matches { + if (m.hash^hash)&m.mask == 0 { + hstr := "" + if hash == 0 { + hstr = "0" + } else { + for ; hash != 0; hash = hash >> 1 { + hstr = string('0'+byte(hash&1)) + hstr + } + } + d.logDebugHashMatchLocked(m.name, "POS="+string(b), hstr, 0) + return true + } + } + return false +} + +// bytesForPos renders a position, including inlining, into d.bytesTmp +// and returns the byte array. d.mu must be locked. +func (d *HashDebug) bytesForPos(ctxt *obj.Link, pos src.XPos) []byte { + d.posTmp = ctxt.AllPos(pos, d.posTmp) + // Reverse posTmp to put outermost first. + b := &d.bytesTmp + b.Reset() + for i := len(d.posTmp) - 1; i >= 0; i-- { + p := &d.posTmp[i] + fmt.Fprintf(b, "%s:%d:%d", p.Filename(), p.Line(), p.Col()) + if i != 0 { + b.WriteByte(';') + } + } + return b.Bytes() +} + +func (d *HashDebug) logDebugHashMatch(varname, name, hstr string, param uint64) { + d.mu.Lock() + defer d.mu.Unlock() + d.logDebugHashMatchLocked(varname, name, hstr, param) +} + +func (d *HashDebug) logDebugHashMatchLocked(varname, name, hstr string, param uint64) { + file := d.logfile + if file == nil { + if tmpfile := os.Getenv("GSHS_LOGFILE"); tmpfile != "" { + var err error + file, err = os.OpenFile(tmpfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + Fatalf("could not open hash-testing logfile %s", tmpfile) + return + } + } + if file == nil { + file = os.Stdout + } + d.logfile = file + } + if len(hstr) > 24 { + hstr = hstr[len(hstr)-24:] + } + // External tools depend on this string + if param == 0 { + fmt.Fprintf(file, "%s triggered %s %s\n", varname, name, hstr) + } else { + fmt.Fprintf(file, "%s triggered %s:%d %s\n", varname, name, param, hstr) + } + file.Sync() +} diff --git a/src/cmd/compile/internal/base/hashdebug_test.go b/src/cmd/compile/internal/base/hashdebug_test.go new file mode 100644 index 0000000..b74169f --- /dev/null +++ b/src/cmd/compile/internal/base/hashdebug_test.go @@ -0,0 +1,164 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package base + +import ( + "bytes" + "strings" + "testing" +) + +func TestHashDebugGossahashY(t *testing.T) { + hd := NewHashDebug("GOSSAHASH", "y", nil) + if hd == nil { + t.Errorf("NewHashDebug should not return nil for GOSSASHASH=y") + } + if !hd.yes { + t.Errorf("NewHashDebug should return hd.yes==true for GOSSASHASH=y") + } + if hd.no { + t.Errorf("NewHashDebug should not return hd.no==true for GOSSASHASH=y") + } +} + +func TestHashDebugGossahashN(t *testing.T) { + hd := NewHashDebug("GOSSAHASH", "n", nil) + if hd == nil { + t.Errorf("NewHashDebug should not return nil for GOSSASHASH=n") + } + if !hd.no { + t.Errorf("NewHashDebug should return hd.no==true GOSSASHASH=n") + } + if hd.yes { + t.Errorf("NewHashDebug should not return hd.yes==true for GOSSASHASH=n") + } +} + +func TestHashDebugGossahashEmpty(t *testing.T) { + hd := NewHashDebug("GOSSAHASH", "", nil) + if hd != nil { + t.Errorf("NewHashDebug should return nil for GOSSASHASH=\"\"") + } +} + +func TestHashDebugMagic(t *testing.T) { + hd := NewHashDebug("FOOXYZZY", "y", nil) + hd0 := NewHashDebug("FOOXYZZY0", "n", nil) + if hd == nil { + t.Errorf("NewHashDebug should have succeeded for FOOXYZZY") + } + if hd0 == nil { + t.Errorf("NewHashDebug should have succeeded for FOOXYZZY0") + } +} + +func TestHash(t *testing.T) { + h0 := hashOf("bar", 0) + h1 := hashOf("bar", 1) + t.Logf(`These values are used in other tests: hashOf("bar,0)"=0x%x, hashOf("bar,1)"=0x%x`, h0, h1) + if h0 == h1 { + t.Errorf("Hashes 0x%x and 0x%x should differ", h0, h1) + } +} + +func TestHashMatch(t *testing.T) { + ws := new(bufferWithSync) + hd := NewHashDebug("GOSSAHASH", "0011", ws) + check := hd.DebugHashMatch("bar") + msg := ws.String() + t.Logf("message was '%s'", msg) + if !check { + t.Errorf("GOSSAHASH=0011 should have matched for 'bar'") + } + wantPrefix(t, msg, "GOSSAHASH triggered bar ") +} + +func TestHashMatchParam(t *testing.T) { + ws := new(bufferWithSync) + hd := NewHashDebug("GOSSAHASH", "1010", ws) + check := hd.DebugHashMatchParam("bar", 1) + msg := ws.String() + t.Logf("message was '%s'", msg) + if !check { + t.Errorf("GOSSAHASH=1010 should have matched for 'bar', 1") + } + wantPrefix(t, msg, "GOSSAHASH triggered bar:1 ") +} + +func TestYMatch(t *testing.T) { + ws := new(bufferWithSync) + hd := NewHashDebug("GOSSAHASH", "y", ws) + check := hd.DebugHashMatch("bar") + msg := ws.String() + t.Logf("message was '%s'", msg) + if !check { + t.Errorf("GOSSAHASH=y should have matched for 'bar'") + } + wantPrefix(t, msg, "GOSSAHASH triggered bar y") +} + +func TestNMatch(t *testing.T) { + ws := new(bufferWithSync) + hd := NewHashDebug("GOSSAHASH", "n", ws) + check := hd.DebugHashMatch("bar") + msg := ws.String() + t.Logf("message was '%s'", msg) + if check { + t.Errorf("GOSSAHASH=n should NOT have matched for 'bar'") + } + if msg != "" { + t.Errorf("Message should have been empty, instead %s", msg) + } +} + +func TestHashNoMatch(t *testing.T) { + ws := new(bufferWithSync) + hd := NewHashDebug("GOSSAHASH", "001100", ws) + check := hd.DebugHashMatch("bar") + msg := ws.String() + t.Logf("message was '%s'", msg) + if check { + t.Errorf("GOSSAHASH=001100 should NOT have matched for 'bar'") + } + if msg != "" { + t.Errorf("Message should have been empty, instead %s", msg) + } + +} + +func TestHashSecondMatch(t *testing.T) { + ws := new(bufferWithSync) + hd := NewHashDebug("GOSSAHASH", "001100/0011", ws) + + check := hd.DebugHashMatch("bar") + msg := ws.String() + t.Logf("message was '%s'", msg) + if !check { + t.Errorf("GOSSAHASH=001100, GOSSAHASH0=0011 should have matched for 'bar'") + } + wantPrefix(t, msg, "GOSSAHASH0 triggered bar") +} + +type bufferWithSync struct { + b bytes.Buffer +} + +func (ws *bufferWithSync) Sync() error { + return nil +} + +func (ws *bufferWithSync) Write(p []byte) (n int, err error) { + return (&ws.b).Write(p) +} + +func (ws *bufferWithSync) String() string { + return strings.TrimSpace((&ws.b).String()) +} + +func wantPrefix(t *testing.T, got, want string) { + if !strings.HasPrefix(got, want) { + t.Errorf("Want %s, got %s", want, got) + } +} diff --git a/src/cmd/compile/internal/base/link.go b/src/cmd/compile/internal/base/link.go new file mode 100644 index 0000000..d8aa5a7 --- /dev/null +++ b/src/cmd/compile/internal/base/link.go @@ -0,0 +1,53 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package base + +import ( + "cmd/internal/obj" +) + +// ReservedImports are import paths used internally for generated +// symbols by the compiler. +// +// The linker uses the magic symbol prefixes "go:" and "type:". +// Avoid potential confusion between import paths and symbols +// by rejecting these reserved imports for now. Also, people +// "can do weird things in GOPATH and we'd prefer they didn't +// do _that_ weird thing" (per rsc). See also #4257. +var ReservedImports = map[string]bool{ + "go": true, + "type": true, +} + +var Ctxt *obj.Link + +// TODO(mdempsky): These should probably be obj.Link methods. + +// PkgLinksym returns the linker symbol for name within the given +// package prefix. For user packages, prefix should be the package +// path encoded with objabi.PathToPrefix. +func PkgLinksym(prefix, name string, abi obj.ABI) *obj.LSym { + if name == "_" { + // TODO(mdempsky): Cleanup callers and Fatalf instead. + return linksym(prefix, "_", abi) + } + sep := "." + if ReservedImports[prefix] { + sep = ":" + } + return linksym(prefix, prefix+sep+name, abi) +} + +// Linkname returns the linker symbol for the given name as it might +// appear within a //go:linkname directive. +func Linkname(name string, abi obj.ABI) *obj.LSym { + return linksym("_", name, abi) +} + +// linksym is an internal helper function for implementing the above +// exported APIs. +func linksym(pkg, name string, abi obj.ABI) *obj.LSym { + return Ctxt.LookupABIInit(name, abi, func(r *obj.LSym) { r.Pkg = pkg }) +} diff --git a/src/cmd/compile/internal/base/mapfile_mmap.go b/src/cmd/compile/internal/base/mapfile_mmap.go new file mode 100644 index 0000000..bbcfda2 --- /dev/null +++ b/src/cmd/compile/internal/base/mapfile_mmap.go @@ -0,0 +1,45 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || (solaris && go1.20) + +package base + +import ( + "internal/unsafeheader" + "os" + "runtime" + "syscall" + "unsafe" +) + +// TODO(mdempsky): Is there a higher-level abstraction that still +// works well for iimport? + +// MapFile returns length bytes from the file starting at the +// specified offset as a string. +func MapFile(f *os.File, offset, length int64) (string, error) { + // POSIX mmap: "The implementation may require that off is a + // multiple of the page size." + x := offset & int64(os.Getpagesize()-1) + offset -= x + length += x + + buf, err := syscall.Mmap(int(f.Fd()), offset, int(length), syscall.PROT_READ, syscall.MAP_SHARED) + runtime.KeepAlive(f) + if err != nil { + return "", err + } + + buf = buf[x:] + pSlice := (*unsafeheader.Slice)(unsafe.Pointer(&buf)) + + var res string + pString := (*unsafeheader.String)(unsafe.Pointer(&res)) + + pString.Data = pSlice.Data + pString.Len = pSlice.Len + + return res, nil +} diff --git a/src/cmd/compile/internal/base/mapfile_read.go b/src/cmd/compile/internal/base/mapfile_read.go new file mode 100644 index 0000000..c1b84db --- /dev/null +++ b/src/cmd/compile/internal/base/mapfile_read.go @@ -0,0 +1,21 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !aix && !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !(solaris && go1.20) + +package base + +import ( + "io" + "os" +) + +func MapFile(f *os.File, offset, length int64) (string, error) { + buf := make([]byte, length) + _, err := io.ReadFull(io.NewSectionReader(f, offset, length), buf) + if err != nil { + return "", err + } + return string(buf), nil +} diff --git a/src/cmd/compile/internal/base/print.go b/src/cmd/compile/internal/base/print.go new file mode 100644 index 0000000..21fa001 --- /dev/null +++ b/src/cmd/compile/internal/base/print.go @@ -0,0 +1,285 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package base + +import ( + "fmt" + "internal/buildcfg" + "os" + "runtime/debug" + "sort" + "strings" + + "cmd/internal/src" +) + +// An errorMsg is a queued error message, waiting to be printed. +type errorMsg struct { + pos src.XPos + msg string +} + +// Pos is the current source position being processed, +// printed by Errorf, ErrorfLang, Fatalf, and Warnf. +var Pos src.XPos + +var ( + errorMsgs []errorMsg + numErrors int // number of entries in errorMsgs that are errors (as opposed to warnings) + numSyntaxErrors int +) + +// Errors returns the number of errors reported. +func Errors() int { + return numErrors +} + +// SyntaxErrors returns the number of syntax errors reported. +func SyntaxErrors() int { + return numSyntaxErrors +} + +// addErrorMsg adds a new errorMsg (which may be a warning) to errorMsgs. +func addErrorMsg(pos src.XPos, format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + // Only add the position if know the position. + // See issue golang.org/issue/11361. + if pos.IsKnown() { + msg = fmt.Sprintf("%v: %s", FmtPos(pos), msg) + } + errorMsgs = append(errorMsgs, errorMsg{ + pos: pos, + msg: msg + "\n", + }) +} + +// FmtPos formats pos as a file:line string. +func FmtPos(pos src.XPos) string { + if Ctxt == nil { + return "???" + } + return Ctxt.OutermostPos(pos).Format(Flag.C == 0, Flag.L == 1) +} + +// byPos sorts errors by source position. +type byPos []errorMsg + +func (x byPos) Len() int { return len(x) } +func (x byPos) Less(i, j int) bool { return x[i].pos.Before(x[j].pos) } +func (x byPos) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + +// FlushErrors sorts errors seen so far by line number, prints them to stdout, +// and empties the errors array. +func FlushErrors() { + if Ctxt != nil && Ctxt.Bso != nil { + Ctxt.Bso.Flush() + } + if len(errorMsgs) == 0 { + return + } + sort.Stable(byPos(errorMsgs)) + for i, err := range errorMsgs { + if i == 0 || err.msg != errorMsgs[i-1].msg { + fmt.Printf("%s", err.msg) + } + } + errorMsgs = errorMsgs[:0] +} + +// lasterror keeps track of the most recently issued error, +// to avoid printing multiple error messages on the same line. +var lasterror struct { + syntax src.XPos // source position of last syntax error + other src.XPos // source position of last non-syntax error + msg string // error message of last non-syntax error +} + +// sameline reports whether two positions a, b are on the same line. +func sameline(a, b src.XPos) bool { + p := Ctxt.PosTable.Pos(a) + q := Ctxt.PosTable.Pos(b) + return p.Base() == q.Base() && p.Line() == q.Line() +} + +// Errorf reports a formatted error at the current line. +func Errorf(format string, args ...interface{}) { + ErrorfAt(Pos, format, args...) +} + +// ErrorfAt reports a formatted error message at pos. +func ErrorfAt(pos src.XPos, format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + + if strings.HasPrefix(msg, "syntax error") { + numSyntaxErrors++ + // only one syntax error per line, no matter what error + if sameline(lasterror.syntax, pos) { + return + } + lasterror.syntax = pos + } else { + // only one of multiple equal non-syntax errors per line + // (FlushErrors shows only one of them, so we filter them + // here as best as we can (they may not appear in order) + // so that we don't count them here and exit early, and + // then have nothing to show for.) + if sameline(lasterror.other, pos) && lasterror.msg == msg { + return + } + lasterror.other = pos + lasterror.msg = msg + } + + addErrorMsg(pos, "%s", msg) + numErrors++ + + hcrash() + if numErrors >= 10 && Flag.LowerE == 0 { + FlushErrors() + fmt.Printf("%v: too many errors\n", FmtPos(pos)) + ErrorExit() + } +} + +// ErrorfVers reports that a language feature (format, args) requires a later version of Go. +func ErrorfVers(lang string, format string, args ...interface{}) { + Errorf("%s requires %s or later (-lang was set to %s; check go.mod)", fmt.Sprintf(format, args...), lang, Flag.Lang) +} + +// UpdateErrorDot is a clumsy hack that rewrites the last error, +// if it was "LINE: undefined: NAME", to be "LINE: undefined: NAME in EXPR". +// It is used to give better error messages for dot (selector) expressions. +func UpdateErrorDot(line string, name, expr string) { + if len(errorMsgs) == 0 { + return + } + e := &errorMsgs[len(errorMsgs)-1] + if strings.HasPrefix(e.msg, line) && e.msg == fmt.Sprintf("%v: undefined: %v\n", line, name) { + e.msg = fmt.Sprintf("%v: undefined: %v in %v\n", line, name, expr) + } +} + +// Warn reports a formatted warning at the current line. +// In general the Go compiler does NOT generate warnings, +// so this should be used only when the user has opted in +// to additional output by setting a particular flag. +func Warn(format string, args ...interface{}) { + WarnfAt(Pos, format, args...) +} + +// WarnfAt reports a formatted warning at pos. +// In general the Go compiler does NOT generate warnings, +// so this should be used only when the user has opted in +// to additional output by setting a particular flag. +func WarnfAt(pos src.XPos, format string, args ...interface{}) { + addErrorMsg(pos, format, args...) + if Flag.LowerM != 0 { + FlushErrors() + } +} + +// Fatalf reports a fatal error - an internal problem - at the current line and exits. +// If other errors have already been printed, then Fatalf just quietly exits. +// (The internal problem may have been caused by incomplete information +// after the already-reported errors, so best to let users fix those and +// try again without being bothered about a spurious internal error.) +// +// But if no errors have been printed, or if -d panic has been specified, +// Fatalf prints the error as an "internal compiler error". In a released build, +// it prints an error asking to file a bug report. In development builds, it +// prints a stack trace. +// +// If -h has been specified, Fatalf panics to force the usual runtime info dump. +func Fatalf(format string, args ...interface{}) { + FatalfAt(Pos, format, args...) +} + +// FatalfAt reports a fatal error - an internal problem - at pos and exits. +// If other errors have already been printed, then FatalfAt just quietly exits. +// (The internal problem may have been caused by incomplete information +// after the already-reported errors, so best to let users fix those and +// try again without being bothered about a spurious internal error.) +// +// But if no errors have been printed, or if -d panic has been specified, +// FatalfAt prints the error as an "internal compiler error". In a released build, +// it prints an error asking to file a bug report. In development builds, it +// prints a stack trace. +// +// If -h has been specified, FatalfAt panics to force the usual runtime info dump. +func FatalfAt(pos src.XPos, format string, args ...interface{}) { + FlushErrors() + + if Debug.Panic != 0 || numErrors == 0 { + fmt.Printf("%v: internal compiler error: ", FmtPos(pos)) + fmt.Printf(format, args...) + fmt.Printf("\n") + + // If this is a released compiler version, ask for a bug report. + if Debug.Panic == 0 && strings.HasPrefix(buildcfg.Version, "go") { + fmt.Printf("\n") + fmt.Printf("Please file a bug report including a short program that triggers the error.\n") + fmt.Printf("https://go.dev/issue/new\n") + } else { + // Not a release; dump a stack trace, too. + fmt.Println() + os.Stdout.Write(debug.Stack()) + fmt.Println() + } + } + + hcrash() + ErrorExit() +} + +// Assert reports "assertion failed" with Fatalf, unless b is true. +func Assert(b bool) { + if !b { + Fatalf("assertion failed") + } +} + +// Assertf reports a fatal error with Fatalf, unless b is true. +func Assertf(b bool, format string, args ...interface{}) { + if !b { + Fatalf(format, args...) + } +} + +// AssertfAt reports a fatal error with FatalfAt, unless b is true. +func AssertfAt(b bool, pos src.XPos, format string, args ...interface{}) { + if !b { + FatalfAt(pos, format, args...) + } +} + +// hcrash crashes the compiler when -h is set, to find out where a message is generated. +func hcrash() { + if Flag.LowerH != 0 { + FlushErrors() + if Flag.LowerO != "" { + os.Remove(Flag.LowerO) + } + panic("-h") + } +} + +// ErrorExit handles an error-status exit. +// It flushes any pending errors, removes the output file, and exits. +func ErrorExit() { + FlushErrors() + if Flag.LowerO != "" { + os.Remove(Flag.LowerO) + } + os.Exit(2) +} + +// ExitIfErrors calls ErrorExit if any errors have been reported. +func ExitIfErrors() { + if Errors() > 0 { + ErrorExit() + } +} + +var AutogeneratedPos src.XPos diff --git a/src/cmd/compile/internal/base/timings.go b/src/cmd/compile/internal/base/timings.go new file mode 100644 index 0000000..f599f4e --- /dev/null +++ b/src/cmd/compile/internal/base/timings.go @@ -0,0 +1,237 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package base + +import ( + "fmt" + "io" + "strings" + "time" +) + +var Timer Timings + +// Timings collects the execution times of labeled phases +// which are added trough a sequence of Start/Stop calls. +// Events may be associated with each phase via AddEvent. +type Timings struct { + list []timestamp + events map[int][]*event // lazily allocated +} + +type timestamp struct { + time time.Time + label string + start bool +} + +type event struct { + size int64 // count or amount of data processed (allocations, data size, lines, funcs, ...) + unit string // unit of size measure (count, MB, lines, funcs, ...) +} + +func (t *Timings) append(labels []string, start bool) { + t.list = append(t.list, timestamp{time.Now(), strings.Join(labels, ":"), start}) +} + +// Start marks the beginning of a new phase and implicitly stops the previous phase. +// The phase name is the colon-separated concatenation of the labels. +func (t *Timings) Start(labels ...string) { + t.append(labels, true) +} + +// Stop marks the end of a phase and implicitly starts a new phase. +// The labels are added to the labels of the ended phase. +func (t *Timings) Stop(labels ...string) { + t.append(labels, false) +} + +// AddEvent associates an event, i.e., a count, or an amount of data, +// with the most recently started or stopped phase; or the very first +// phase if Start or Stop hasn't been called yet. The unit specifies +// the unit of measurement (e.g., MB, lines, no. of funcs, etc.). +func (t *Timings) AddEvent(size int64, unit string) { + m := t.events + if m == nil { + m = make(map[int][]*event) + t.events = m + } + i := len(t.list) + if i > 0 { + i-- + } + m[i] = append(m[i], &event{size, unit}) +} + +// Write prints the phase times to w. +// The prefix is printed at the start of each line. +func (t *Timings) Write(w io.Writer, prefix string) { + if len(t.list) > 0 { + var lines lines + + // group of phases with shared non-empty label prefix + var group struct { + label string // label prefix + tot time.Duration // accumulated phase time + size int // number of phases collected in group + } + + // accumulated time between Stop/Start timestamps + var unaccounted time.Duration + + // process Start/Stop timestamps + pt := &t.list[0] // previous timestamp + tot := t.list[len(t.list)-1].time.Sub(pt.time) + for i := 1; i < len(t.list); i++ { + qt := &t.list[i] // current timestamp + dt := qt.time.Sub(pt.time) + + var label string + var events []*event + if pt.start { + // previous phase started + label = pt.label + events = t.events[i-1] + if qt.start { + // start implicitly ended previous phase; nothing to do + } else { + // stop ended previous phase; append stop labels, if any + if qt.label != "" { + label += ":" + qt.label + } + // events associated with stop replace prior events + if e := t.events[i]; e != nil { + events = e + } + } + } else { + // previous phase stopped + if qt.start { + // between a stopped and started phase; unaccounted time + unaccounted += dt + } else { + // previous stop implicitly started current phase + label = qt.label + events = t.events[i] + } + } + if label != "" { + // add phase to existing group, or start a new group + l := commonPrefix(group.label, label) + if group.size == 1 && l != "" || group.size > 1 && l == group.label { + // add to existing group + group.label = l + group.tot += dt + group.size++ + } else { + // start a new group + if group.size > 1 { + lines.add(prefix+group.label+"subtotal", 1, group.tot, tot, nil) + } + group.label = label + group.tot = dt + group.size = 1 + } + + // write phase + lines.add(prefix+label, 1, dt, tot, events) + } + + pt = qt + } + + if group.size > 1 { + lines.add(prefix+group.label+"subtotal", 1, group.tot, tot, nil) + } + + if unaccounted != 0 { + lines.add(prefix+"unaccounted", 1, unaccounted, tot, nil) + } + + lines.add(prefix+"total", 1, tot, tot, nil) + + lines.write(w) + } +} + +func commonPrefix(a, b string) string { + i := 0 + for i < len(a) && i < len(b) && a[i] == b[i] { + i++ + } + return a[:i] +} + +type lines [][]string + +func (lines *lines) add(label string, n int, dt, tot time.Duration, events []*event) { + var line []string + add := func(format string, args ...interface{}) { + line = append(line, fmt.Sprintf(format, args...)) + } + + add("%s", label) + add(" %d", n) + add(" %d ns/op", dt) + add(" %.2f %%", float64(dt)/float64(tot)*100) + + for _, e := range events { + add(" %d", e.size) + add(" %s", e.unit) + add(" %d", int64(float64(e.size)/dt.Seconds()+0.5)) + add(" %s/s", e.unit) + } + + *lines = append(*lines, line) +} + +func (lines lines) write(w io.Writer) { + // determine column widths and contents + var widths []int + var number []bool + for _, line := range lines { + for i, col := range line { + if i < len(widths) { + if len(col) > widths[i] { + widths[i] = len(col) + } + } else { + widths = append(widths, len(col)) + number = append(number, isnumber(col)) // first line determines column contents + } + } + } + + // make column widths a multiple of align for more stable output + const align = 1 // set to a value > 1 to enable + if align > 1 { + for i, w := range widths { + w += align - 1 + widths[i] = w - w%align + } + } + + // print lines taking column widths and contents into account + for _, line := range lines { + for i, col := range line { + format := "%-*s" + if number[i] { + format = "%*s" // numbers are right-aligned + } + fmt.Fprintf(w, format, widths[i], col) + } + fmt.Fprintln(w) + } +} + +func isnumber(s string) bool { + for _, ch := range s { + if ch <= ' ' { + continue // ignore leading whitespace + } + return '0' <= ch && ch <= '9' || ch == '.' || ch == '-' || ch == '+' + } + return false +} |