From 73df946d56c74384511a194dd01dbe099584fd1a Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 28 Apr 2024 15:14:23 +0200 Subject: Adding upstream version 1.16.10. Signed-off-by: Daniel Baumann --- src/cmd/go/internal/work/action.go | 859 ++++++++ src/cmd/go/internal/work/build.go | 929 ++++++++ src/cmd/go/internal/work/build_test.go | 277 +++ src/cmd/go/internal/work/buildid.go | 698 ++++++ src/cmd/go/internal/work/exec.go | 3316 +++++++++++++++++++++++++++++ src/cmd/go/internal/work/exec_test.go | 86 + src/cmd/go/internal/work/gc.go | 693 ++++++ src/cmd/go/internal/work/gccgo.go | 618 ++++++ src/cmd/go/internal/work/init.go | 286 +++ src/cmd/go/internal/work/security.go | 310 +++ src/cmd/go/internal/work/security_test.go | 277 +++ src/cmd/go/internal/work/testgo.go | 48 + 12 files changed, 8397 insertions(+) create mode 100644 src/cmd/go/internal/work/action.go create mode 100644 src/cmd/go/internal/work/build.go create mode 100644 src/cmd/go/internal/work/build_test.go create mode 100644 src/cmd/go/internal/work/buildid.go create mode 100644 src/cmd/go/internal/work/exec.go create mode 100644 src/cmd/go/internal/work/exec_test.go create mode 100644 src/cmd/go/internal/work/gc.go create mode 100644 src/cmd/go/internal/work/gccgo.go create mode 100644 src/cmd/go/internal/work/init.go create mode 100644 src/cmd/go/internal/work/security.go create mode 100644 src/cmd/go/internal/work/security_test.go create mode 100644 src/cmd/go/internal/work/testgo.go (limited to 'src/cmd/go/internal/work') diff --git a/src/cmd/go/internal/work/action.go b/src/cmd/go/internal/work/action.go new file mode 100644 index 0000000..9d141ae --- /dev/null +++ b/src/cmd/go/internal/work/action.go @@ -0,0 +1,859 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Action graph creation (planning). + +package work + +import ( + "bufio" + "bytes" + "container/heap" + "context" + "debug/elf" + "encoding/json" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "time" + + "cmd/go/internal/base" + "cmd/go/internal/cache" + "cmd/go/internal/cfg" + "cmd/go/internal/load" + "cmd/go/internal/trace" + "cmd/internal/buildid" +) + +// A Builder holds global state about a build. +// It does not hold per-package state, because we +// build packages in parallel, and the builder is shared. +type Builder struct { + WorkDir string // the temporary work directory (ends in filepath.Separator) + actionCache map[cacheKey]*Action // a cache of already-constructed actions + mkdirCache map[string]bool // a cache of created directories + flagCache map[[2]string]bool // a cache of supported compiler flags + Print func(args ...interface{}) (int, error) + + IsCmdList bool // running as part of go list; set p.Stale and additional fields below + NeedError bool // list needs p.Error + NeedExport bool // list needs p.Export + NeedCompiledGoFiles bool // list needs p.CompiledGoFiles + + objdirSeq int // counter for NewObjdir + pkgSeq int + + output sync.Mutex + scriptDir string // current directory in printed script + + exec sync.Mutex + readySema chan bool + ready actionQueue + + id sync.Mutex + toolIDCache map[string]string // tool name -> tool ID + buildIDCache map[string]string // file name -> build ID +} + +// NOTE: Much of Action would not need to be exported if not for test. +// Maybe test functionality should move into this package too? + +// An Action represents a single action in the action graph. +type Action struct { + Mode string // description of action operation + Package *load.Package // the package this action works on + Deps []*Action // actions that must happen before this one + Func func(*Builder, context.Context, *Action) error // the action itself (nil = no-op) + IgnoreFail bool // whether to run f even if dependencies fail + TestOutput *bytes.Buffer // test output buffer + Args []string // additional args for runProgram + + triggers []*Action // inverse of deps + + buggyInstall bool // is this a buggy install (see -linkshared)? + + TryCache func(*Builder, *Action) bool // callback for cache bypass + + // Generated files, directories. + Objdir string // directory for intermediate objects + Target string // goal of the action: the created package or executable + built string // the actual created package or executable + actionID cache.ActionID // cache ID of action input + buildID string // build ID of action output + + VetxOnly bool // Mode=="vet": only being called to supply info about dependencies + needVet bool // Mode=="build": need to fill in vet config + needBuild bool // Mode=="build": need to do actual build (can be false if needVet is true) + vetCfg *vetConfig // vet config + output []byte // output redirect buffer (nil means use b.Print) + + // Execution state. + pending int // number of deps yet to complete + priority int // relative execution priority + Failed bool // whether the action failed + json *actionJSON // action graph information + nonGoOverlay map[string]string // map from non-.go source files to copied files in objdir. Nil if no overlay is used. + traceSpan *trace.Span +} + +// BuildActionID returns the action ID section of a's build ID. +func (a *Action) BuildActionID() string { return actionID(a.buildID) } + +// BuildContentID returns the content ID section of a's build ID. +func (a *Action) BuildContentID() string { return contentID(a.buildID) } + +// BuildID returns a's build ID. +func (a *Action) BuildID() string { return a.buildID } + +// BuiltTarget returns the actual file that was built. This differs +// from Target when the result was cached. +func (a *Action) BuiltTarget() string { return a.built } + +// An actionQueue is a priority queue of actions. +type actionQueue []*Action + +// Implement heap.Interface +func (q *actionQueue) Len() int { return len(*q) } +func (q *actionQueue) Swap(i, j int) { (*q)[i], (*q)[j] = (*q)[j], (*q)[i] } +func (q *actionQueue) Less(i, j int) bool { return (*q)[i].priority < (*q)[j].priority } +func (q *actionQueue) Push(x interface{}) { *q = append(*q, x.(*Action)) } +func (q *actionQueue) Pop() interface{} { + n := len(*q) - 1 + x := (*q)[n] + *q = (*q)[:n] + return x +} + +func (q *actionQueue) push(a *Action) { + if a.json != nil { + a.json.TimeReady = time.Now() + } + heap.Push(q, a) +} + +func (q *actionQueue) pop() *Action { + return heap.Pop(q).(*Action) +} + +type actionJSON struct { + ID int + Mode string + Package string + Deps []int `json:",omitempty"` + IgnoreFail bool `json:",omitempty"` + Args []string `json:",omitempty"` + Link bool `json:",omitempty"` + Objdir string `json:",omitempty"` + Target string `json:",omitempty"` + Priority int `json:",omitempty"` + Failed bool `json:",omitempty"` + Built string `json:",omitempty"` + VetxOnly bool `json:",omitempty"` + NeedVet bool `json:",omitempty"` + NeedBuild bool `json:",omitempty"` + ActionID string `json:",omitempty"` + BuildID string `json:",omitempty"` + TimeReady time.Time `json:",omitempty"` + TimeStart time.Time `json:",omitempty"` + TimeDone time.Time `json:",omitempty"` + + Cmd []string // `json:",omitempty"` + CmdReal time.Duration `json:",omitempty"` + CmdUser time.Duration `json:",omitempty"` + CmdSys time.Duration `json:",omitempty"` +} + +// cacheKey is the key for the action cache. +type cacheKey struct { + mode string + p *load.Package +} + +func actionGraphJSON(a *Action) string { + var workq []*Action + var inWorkq = make(map[*Action]int) + + add := func(a *Action) { + if _, ok := inWorkq[a]; ok { + return + } + inWorkq[a] = len(workq) + workq = append(workq, a) + } + add(a) + + for i := 0; i < len(workq); i++ { + for _, dep := range workq[i].Deps { + add(dep) + } + } + + var list []*actionJSON + for id, a := range workq { + if a.json == nil { + a.json = &actionJSON{ + Mode: a.Mode, + ID: id, + IgnoreFail: a.IgnoreFail, + Args: a.Args, + Objdir: a.Objdir, + Target: a.Target, + Failed: a.Failed, + Priority: a.priority, + Built: a.built, + VetxOnly: a.VetxOnly, + NeedBuild: a.needBuild, + NeedVet: a.needVet, + } + if a.Package != nil { + // TODO(rsc): Make this a unique key for a.Package somehow. + a.json.Package = a.Package.ImportPath + } + for _, a1 := range a.Deps { + a.json.Deps = append(a.json.Deps, inWorkq[a1]) + } + } + list = append(list, a.json) + } + + js, err := json.MarshalIndent(list, "", "\t") + if err != nil { + fmt.Fprintf(os.Stderr, "go: writing debug action graph: %v\n", err) + return "" + } + return string(js) +} + +// BuildMode specifies the build mode: +// are we just building things or also installing the results? +type BuildMode int + +const ( + ModeBuild BuildMode = iota + ModeInstall + ModeBuggyInstall + + ModeVetOnly = 1 << 8 +) + +func (b *Builder) Init() { + b.Print = func(a ...interface{}) (int, error) { + return fmt.Fprint(os.Stderr, a...) + } + b.actionCache = make(map[cacheKey]*Action) + b.mkdirCache = make(map[string]bool) + b.toolIDCache = make(map[string]string) + b.buildIDCache = make(map[string]string) + + if cfg.BuildN { + b.WorkDir = "$WORK" + } else { + tmp, err := os.MkdirTemp(cfg.Getenv("GOTMPDIR"), "go-build") + if err != nil { + base.Fatalf("go: creating work dir: %v", err) + } + if !filepath.IsAbs(tmp) { + abs, err := filepath.Abs(tmp) + if err != nil { + os.RemoveAll(tmp) + base.Fatalf("go: creating work dir: %v", err) + } + tmp = abs + } + b.WorkDir = tmp + if cfg.BuildX || cfg.BuildWork { + fmt.Fprintf(os.Stderr, "WORK=%s\n", b.WorkDir) + } + if !cfg.BuildWork { + workdir := b.WorkDir + base.AtExit(func() { + start := time.Now() + for { + err := os.RemoveAll(workdir) + if err == nil { + return + } + + // On some configurations of Windows, directories containing executable + // files may be locked for a while after the executable exits (perhaps + // due to antivirus scans?). It's probably worth a little extra latency + // on exit to avoid filling up the user's temporary directory with leaked + // files. (See golang.org/issue/30789.) + if runtime.GOOS != "windows" || time.Since(start) >= 500*time.Millisecond { + fmt.Fprintf(os.Stderr, "go: failed to remove work dir: %s\n", err) + return + } + time.Sleep(5 * time.Millisecond) + } + }) + } + } + + if err := CheckGOOSARCHPair(cfg.Goos, cfg.Goarch); err != nil { + fmt.Fprintf(os.Stderr, "cmd/go: %v\n", err) + base.SetExitStatus(2) + base.Exit() + } + + for _, tag := range cfg.BuildContext.BuildTags { + if strings.Contains(tag, ",") { + fmt.Fprintf(os.Stderr, "cmd/go: -tags space-separated list contains comma\n") + base.SetExitStatus(2) + base.Exit() + } + } +} + +func CheckGOOSARCHPair(goos, goarch string) error { + if _, ok := cfg.OSArchSupportsCgo[goos+"/"+goarch]; !ok && cfg.BuildContext.Compiler == "gc" { + return fmt.Errorf("unsupported GOOS/GOARCH pair %s/%s", goos, goarch) + } + return nil +} + +// NewObjdir returns the name of a fresh object directory under b.WorkDir. +// It is up to the caller to call b.Mkdir on the result at an appropriate time. +// The result ends in a slash, so that file names in that directory +// can be constructed with direct string addition. +// +// NewObjdir must be called only from a single goroutine at a time, +// so it is safe to call during action graph construction, but it must not +// be called during action graph execution. +func (b *Builder) NewObjdir() string { + b.objdirSeq++ + return filepath.Join(b.WorkDir, fmt.Sprintf("b%03d", b.objdirSeq)) + string(filepath.Separator) +} + +// readpkglist returns the list of packages that were built into the shared library +// at shlibpath. For the native toolchain this list is stored, newline separated, in +// an ELF note with name "Go\x00\x00" and type 1. For GCCGO it is extracted from the +// .go_export section. +func readpkglist(shlibpath string) (pkgs []*load.Package) { + var stk load.ImportStack + if cfg.BuildToolchainName == "gccgo" { + f, _ := elf.Open(shlibpath) + sect := f.Section(".go_export") + data, _ := sect.Data() + scanner := bufio.NewScanner(bytes.NewBuffer(data)) + for scanner.Scan() { + t := scanner.Text() + if strings.HasPrefix(t, "pkgpath ") { + t = strings.TrimPrefix(t, "pkgpath ") + t = strings.TrimSuffix(t, ";") + pkgs = append(pkgs, load.LoadImportWithFlags(t, base.Cwd, nil, &stk, nil, 0)) + } + } + } else { + pkglistbytes, err := buildid.ReadELFNote(shlibpath, "Go\x00\x00", 1) + if err != nil { + base.Fatalf("readELFNote failed: %v", err) + } + scanner := bufio.NewScanner(bytes.NewBuffer(pkglistbytes)) + for scanner.Scan() { + t := scanner.Text() + pkgs = append(pkgs, load.LoadImportWithFlags(t, base.Cwd, nil, &stk, nil, 0)) + } + } + return +} + +// cacheAction looks up {mode, p} in the cache and returns the resulting action. +// If the cache has no such action, f() is recorded and returned. +// TODO(rsc): Change the second key from *load.Package to interface{}, +// to make the caching in linkShared less awkward? +func (b *Builder) cacheAction(mode string, p *load.Package, f func() *Action) *Action { + a := b.actionCache[cacheKey{mode, p}] + if a == nil { + a = f() + b.actionCache[cacheKey{mode, p}] = a + } + return a +} + +// AutoAction returns the "right" action for go build or go install of p. +func (b *Builder) AutoAction(mode, depMode BuildMode, p *load.Package) *Action { + if p.Name == "main" { + return b.LinkAction(mode, depMode, p) + } + return b.CompileAction(mode, depMode, p) +} + +// CompileAction returns the action for compiling and possibly installing +// (according to mode) the given package. The resulting action is only +// for building packages (archives), never for linking executables. +// depMode is the action (build or install) to use when building dependencies. +// To turn package main into an executable, call b.Link instead. +func (b *Builder) CompileAction(mode, depMode BuildMode, p *load.Package) *Action { + vetOnly := mode&ModeVetOnly != 0 + mode &^= ModeVetOnly + + if mode != ModeBuild && (p.Internal.Local || p.Module != nil) && p.Target == "" { + // Imported via local path or using modules. No permanent target. + mode = ModeBuild + } + if mode != ModeBuild && p.Name == "main" { + // We never install the .a file for a main package. + mode = ModeBuild + } + + // Construct package build action. + a := b.cacheAction("build", p, func() *Action { + a := &Action{ + Mode: "build", + Package: p, + Func: (*Builder).build, + Objdir: b.NewObjdir(), + } + + if p.Error == nil || !p.Error.IsImportCycle { + for _, p1 := range p.Internal.Imports { + a.Deps = append(a.Deps, b.CompileAction(depMode, depMode, p1)) + } + } + + if p.Standard { + switch p.ImportPath { + case "builtin", "unsafe": + // Fake packages - nothing to build. + a.Mode = "built-in package" + a.Func = nil + return a + } + + // gccgo standard library is "fake" too. + if cfg.BuildToolchainName == "gccgo" { + // the target name is needed for cgo. + a.Mode = "gccgo stdlib" + a.Target = p.Target + a.Func = nil + return a + } + } + + return a + }) + + // Find the build action; the cache entry may have been replaced + // by the install action during (*Builder).installAction. + buildAction := a + switch buildAction.Mode { + case "build", "built-in package", "gccgo stdlib": + // ok + case "build-install": + buildAction = a.Deps[0] + default: + panic("lost build action: " + buildAction.Mode) + } + buildAction.needBuild = buildAction.needBuild || !vetOnly + + // Construct install action. + if mode == ModeInstall || mode == ModeBuggyInstall { + a = b.installAction(a, mode) + } + + return a +} + +// VetAction returns the action for running go vet on package p. +// It depends on the action for compiling p. +// If the caller may be causing p to be installed, it is up to the caller +// to make sure that the install depends on (runs after) vet. +func (b *Builder) VetAction(mode, depMode BuildMode, p *load.Package) *Action { + a := b.vetAction(mode, depMode, p) + a.VetxOnly = false + return a +} + +func (b *Builder) vetAction(mode, depMode BuildMode, p *load.Package) *Action { + // Construct vet action. + a := b.cacheAction("vet", p, func() *Action { + a1 := b.CompileAction(mode|ModeVetOnly, depMode, p) + + // vet expects to be able to import "fmt". + var stk load.ImportStack + stk.Push("vet") + p1 := load.LoadImportWithFlags("fmt", p.Dir, p, &stk, nil, 0) + stk.Pop() + aFmt := b.CompileAction(ModeBuild, depMode, p1) + + var deps []*Action + if a1.buggyInstall { + // (*Builder).vet expects deps[0] to be the package + // and deps[1] to be "fmt". If we see buggyInstall + // here then a1 is an install of a shared library, + // and the real package is a1.Deps[0]. + deps = []*Action{a1.Deps[0], aFmt, a1} + } else { + deps = []*Action{a1, aFmt} + } + for _, p1 := range p.Internal.Imports { + deps = append(deps, b.vetAction(mode, depMode, p1)) + } + + a := &Action{ + Mode: "vet", + Package: p, + Deps: deps, + Objdir: a1.Objdir, + VetxOnly: true, + IgnoreFail: true, // it's OK if vet of dependencies "fails" (reports problems) + } + if a1.Func == nil { + // Built-in packages like unsafe. + return a + } + deps[0].needVet = true + a.Func = (*Builder).vet + return a + }) + return a +} + +// LinkAction returns the action for linking p into an executable +// and possibly installing the result (according to mode). +// depMode is the action (build or install) to use when compiling dependencies. +func (b *Builder) LinkAction(mode, depMode BuildMode, p *load.Package) *Action { + // Construct link action. + a := b.cacheAction("link", p, func() *Action { + a := &Action{ + Mode: "link", + Package: p, + } + + a1 := b.CompileAction(ModeBuild, depMode, p) + a.Func = (*Builder).link + a.Deps = []*Action{a1} + a.Objdir = a1.Objdir + + // An executable file. (This is the name of a temporary file.) + // Because we run the temporary file in 'go run' and 'go test', + // the name will show up in ps listings. If the caller has specified + // a name, use that instead of a.out. The binary is generated + // in an otherwise empty subdirectory named exe to avoid + // naming conflicts. The only possible conflict is if we were + // to create a top-level package named exe. + name := "a.out" + if p.Internal.ExeName != "" { + name = p.Internal.ExeName + } else if (cfg.Goos == "darwin" || cfg.Goos == "windows") && cfg.BuildBuildmode == "c-shared" && p.Target != "" { + // On OS X, the linker output name gets recorded in the + // shared library's LC_ID_DYLIB load command. + // The code invoking the linker knows to pass only the final + // path element. Arrange that the path element matches what + // we'll install it as; otherwise the library is only loadable as "a.out". + // On Windows, DLL file name is recorded in PE file + // export section, so do like on OS X. + _, name = filepath.Split(p.Target) + } + a.Target = a.Objdir + filepath.Join("exe", name) + cfg.ExeSuffix + a.built = a.Target + b.addTransitiveLinkDeps(a, a1, "") + + // Sequence the build of the main package (a1) strictly after the build + // of all other dependencies that go into the link. It is likely to be after + // them anyway, but just make sure. This is required by the build ID-based + // shortcut in (*Builder).useCache(a1), which will call b.linkActionID(a). + // In order for that linkActionID call to compute the right action ID, all the + // dependencies of a (except a1) must have completed building and have + // recorded their build IDs. + a1.Deps = append(a1.Deps, &Action{Mode: "nop", Deps: a.Deps[1:]}) + return a + }) + + if mode == ModeInstall || mode == ModeBuggyInstall { + a = b.installAction(a, mode) + } + + return a +} + +// installAction returns the action for installing the result of a1. +func (b *Builder) installAction(a1 *Action, mode BuildMode) *Action { + // Because we overwrite the build action with the install action below, + // a1 may already be an install action fetched from the "build" cache key, + // and the caller just doesn't realize. + if strings.HasSuffix(a1.Mode, "-install") { + if a1.buggyInstall && mode == ModeInstall { + // Congratulations! The buggy install is now a proper install. + a1.buggyInstall = false + } + return a1 + } + + // If there's no actual action to build a1, + // there's nothing to install either. + // This happens if a1 corresponds to reusing an already-built object. + if a1.Func == nil { + return a1 + } + + p := a1.Package + return b.cacheAction(a1.Mode+"-install", p, func() *Action { + // The install deletes the temporary build result, + // so we need all other actions, both past and future, + // that attempt to depend on the build to depend instead + // on the install. + + // Make a private copy of a1 (the build action), + // no longer accessible to any other rules. + buildAction := new(Action) + *buildAction = *a1 + + // Overwrite a1 with the install action. + // This takes care of updating past actions that + // point at a1 for the build action; now they will + // point at a1 and get the install action. + // We also leave a1 in the action cache as the result + // for "build", so that actions not yet created that + // try to depend on the build will instead depend + // on the install. + *a1 = Action{ + Mode: buildAction.Mode + "-install", + Func: BuildInstallFunc, + Package: p, + Objdir: buildAction.Objdir, + Deps: []*Action{buildAction}, + Target: p.Target, + built: p.Target, + + buggyInstall: mode == ModeBuggyInstall, + } + + b.addInstallHeaderAction(a1) + return a1 + }) +} + +// addTransitiveLinkDeps adds to the link action a all packages +// that are transitive dependencies of a1.Deps. +// That is, if a is a link of package main, a1 is the compile of package main +// and a1.Deps is the actions for building packages directly imported by +// package main (what the compiler needs). The linker needs all packages +// transitively imported by the whole program; addTransitiveLinkDeps +// makes sure those are present in a.Deps. +// If shlib is non-empty, then a corresponds to the build and installation of shlib, +// so any rebuild of shlib should not be added as a dependency. +func (b *Builder) addTransitiveLinkDeps(a, a1 *Action, shlib string) { + // Expand Deps to include all built packages, for the linker. + // Use breadth-first search to find rebuilt-for-test packages + // before the standard ones. + // TODO(rsc): Eliminate the standard ones from the action graph, + // which will require doing a little bit more rebuilding. + workq := []*Action{a1} + haveDep := map[string]bool{} + if a1.Package != nil { + haveDep[a1.Package.ImportPath] = true + } + for i := 0; i < len(workq); i++ { + a1 := workq[i] + for _, a2 := range a1.Deps { + // TODO(rsc): Find a better discriminator than the Mode strings, once the dust settles. + if a2.Package == nil || (a2.Mode != "build-install" && a2.Mode != "build") || haveDep[a2.Package.ImportPath] { + continue + } + haveDep[a2.Package.ImportPath] = true + a.Deps = append(a.Deps, a2) + if a2.Mode == "build-install" { + a2 = a2.Deps[0] // walk children of "build" action + } + workq = append(workq, a2) + } + } + + // If this is go build -linkshared, then the link depends on the shared libraries + // in addition to the packages themselves. (The compile steps do not.) + if cfg.BuildLinkshared { + haveShlib := map[string]bool{shlib: true} + for _, a1 := range a.Deps { + p1 := a1.Package + if p1 == nil || p1.Shlib == "" || haveShlib[filepath.Base(p1.Shlib)] { + continue + } + haveShlib[filepath.Base(p1.Shlib)] = true + // TODO(rsc): The use of ModeInstall here is suspect, but if we only do ModeBuild, + // we'll end up building an overall library or executable that depends at runtime + // on other libraries that are out-of-date, which is clearly not good either. + // We call it ModeBuggyInstall to make clear that this is not right. + a.Deps = append(a.Deps, b.linkSharedAction(ModeBuggyInstall, ModeBuggyInstall, p1.Shlib, nil)) + } + } +} + +// addInstallHeaderAction adds an install header action to a, if needed. +// The action a should be an install action as generated by either +// b.CompileAction or b.LinkAction with mode=ModeInstall, +// and so a.Deps[0] is the corresponding build action. +func (b *Builder) addInstallHeaderAction(a *Action) { + // Install header for cgo in c-archive and c-shared modes. + p := a.Package + if p.UsesCgo() && (cfg.BuildBuildmode == "c-archive" || cfg.BuildBuildmode == "c-shared") { + hdrTarget := a.Target[:len(a.Target)-len(filepath.Ext(a.Target))] + ".h" + if cfg.BuildContext.Compiler == "gccgo" && cfg.BuildO == "" { + // For the header file, remove the "lib" + // added by go/build, so we generate pkg.h + // rather than libpkg.h. + dir, file := filepath.Split(hdrTarget) + file = strings.TrimPrefix(file, "lib") + hdrTarget = filepath.Join(dir, file) + } + ah := &Action{ + Mode: "install header", + Package: a.Package, + Deps: []*Action{a.Deps[0]}, + Func: (*Builder).installHeader, + Objdir: a.Deps[0].Objdir, + Target: hdrTarget, + } + a.Deps = append(a.Deps, ah) + } +} + +// buildmodeShared takes the "go build" action a1 into the building of a shared library of a1.Deps. +// That is, the input a1 represents "go build pkgs" and the result represents "go build -buildmode=shared pkgs". +func (b *Builder) buildmodeShared(mode, depMode BuildMode, args []string, pkgs []*load.Package, a1 *Action) *Action { + name, err := libname(args, pkgs) + if err != nil { + base.Fatalf("%v", err) + } + return b.linkSharedAction(mode, depMode, name, a1) +} + +// linkSharedAction takes a grouping action a1 corresponding to a list of built packages +// and returns an action that links them together into a shared library with the name shlib. +// If a1 is nil, shlib should be an absolute path to an existing shared library, +// and then linkSharedAction reads that library to find out the package list. +func (b *Builder) linkSharedAction(mode, depMode BuildMode, shlib string, a1 *Action) *Action { + fullShlib := shlib + shlib = filepath.Base(shlib) + a := b.cacheAction("build-shlib "+shlib, nil, func() *Action { + if a1 == nil { + // TODO(rsc): Need to find some other place to store config, + // not in pkg directory. See golang.org/issue/22196. + pkgs := readpkglist(fullShlib) + a1 = &Action{ + Mode: "shlib packages", + } + for _, p := range pkgs { + a1.Deps = append(a1.Deps, b.CompileAction(mode, depMode, p)) + } + } + + // Fake package to hold ldflags. + // As usual shared libraries are a kludgy, abstraction-violating special case: + // we let them use the flags specified for the command-line arguments. + p := &load.Package{} + p.Internal.CmdlinePkg = true + p.Internal.Ldflags = load.BuildLdflags.For(p) + p.Internal.Gccgoflags = load.BuildGccgoflags.For(p) + + // Add implicit dependencies to pkgs list. + // Currently buildmode=shared forces external linking mode, and + // external linking mode forces an import of runtime/cgo (and + // math on arm). So if it was not passed on the command line and + // it is not present in another shared library, add it here. + // TODO(rsc): Maybe this should only happen if "runtime" is in the original package set. + // TODO(rsc): This should probably be changed to use load.LinkerDeps(p). + // TODO(rsc): We don't add standard library imports for gccgo + // because they are all always linked in anyhow. + // Maybe load.LinkerDeps should be used and updated. + a := &Action{ + Mode: "go build -buildmode=shared", + Package: p, + Objdir: b.NewObjdir(), + Func: (*Builder).linkShared, + Deps: []*Action{a1}, + } + a.Target = filepath.Join(a.Objdir, shlib) + if cfg.BuildToolchainName != "gccgo" { + add := func(a1 *Action, pkg string, force bool) { + for _, a2 := range a1.Deps { + if a2.Package != nil && a2.Package.ImportPath == pkg { + return + } + } + var stk load.ImportStack + p := load.LoadImportWithFlags(pkg, base.Cwd, nil, &stk, nil, 0) + if p.Error != nil { + base.Fatalf("load %s: %v", pkg, p.Error) + } + // Assume that if pkg (runtime/cgo or math) + // is already accounted for in a different shared library, + // then that shared library also contains runtime, + // so that anything we do will depend on that library, + // so we don't need to include pkg in our shared library. + if force || p.Shlib == "" || filepath.Base(p.Shlib) == pkg { + a1.Deps = append(a1.Deps, b.CompileAction(depMode, depMode, p)) + } + } + add(a1, "runtime/cgo", false) + if cfg.Goarch == "arm" { + add(a1, "math", false) + } + + // The linker step still needs all the usual linker deps. + // (For example, the linker always opens runtime.a.) + for _, dep := range load.LinkerDeps(nil) { + add(a, dep, true) + } + } + b.addTransitiveLinkDeps(a, a1, shlib) + return a + }) + + // Install result. + if (mode == ModeInstall || mode == ModeBuggyInstall) && a.Func != nil { + buildAction := a + + a = b.cacheAction("install-shlib "+shlib, nil, func() *Action { + // Determine the eventual install target. + // The install target is root/pkg/shlib, where root is the source root + // in which all the packages lie. + // TODO(rsc): Perhaps this cross-root check should apply to the full + // transitive package dependency list, not just the ones named + // on the command line? + pkgDir := a1.Deps[0].Package.Internal.Build.PkgTargetRoot + for _, a2 := range a1.Deps { + if dir := a2.Package.Internal.Build.PkgTargetRoot; dir != pkgDir { + base.Fatalf("installing shared library: cannot use packages %s and %s from different roots %s and %s", + a1.Deps[0].Package.ImportPath, + a2.Package.ImportPath, + pkgDir, + dir) + } + } + // TODO(rsc): Find out and explain here why gccgo is different. + if cfg.BuildToolchainName == "gccgo" { + pkgDir = filepath.Join(pkgDir, "shlibs") + } + target := filepath.Join(pkgDir, shlib) + + a := &Action{ + Mode: "go install -buildmode=shared", + Objdir: buildAction.Objdir, + Func: BuildInstallFunc, + Deps: []*Action{buildAction}, + Target: target, + } + for _, a2 := range buildAction.Deps[0].Deps { + p := a2.Package + if p.Target == "" { + continue + } + a.Deps = append(a.Deps, &Action{ + Mode: "shlibname", + Package: p, + Func: (*Builder).installShlibname, + Target: strings.TrimSuffix(p.Target, ".a") + ".shlibname", + Deps: []*Action{a.Deps[0]}, + }) + } + return a + }) + } + + return a +} diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go new file mode 100644 index 0000000..f024b07 --- /dev/null +++ b/src/cmd/go/internal/work/build.go @@ -0,0 +1,929 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "context" + "errors" + "fmt" + "go/build" + exec "internal/execabs" + "internal/goroot" + "os" + "path" + "path/filepath" + "runtime" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/load" + "cmd/go/internal/modfetch" + "cmd/go/internal/modload" + "cmd/go/internal/search" + "cmd/go/internal/trace" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" +) + +var CmdBuild = &base.Command{ + UsageLine: "go build [-o output] [build flags] [packages]", + Short: "compile packages and dependencies", + Long: ` +Build compiles the packages named by the import paths, +along with their dependencies, but it does not install the results. + +If the arguments to build are a list of .go files from a single directory, +build treats them as a list of source files specifying a single package. + +When compiling packages, build ignores files that end in '_test.go'. + +When compiling a single main package, build writes +the resulting executable to an output file named after +the first source file ('go build ed.go rx.go' writes 'ed' or 'ed.exe') +or the source code directory ('go build unix/sam' writes 'sam' or 'sam.exe'). +The '.exe' suffix is added when writing a Windows executable. + +When compiling multiple packages or a single non-main package, +build compiles the packages but discards the resulting object, +serving only as a check that the packages can be built. + +The -o flag forces build to write the resulting executable or object +to the named output file or directory, instead of the default behavior described +in the last two paragraphs. If the named output is an existing directory or +ends with a slash or backslash, then any resulting executables +will be written to that directory. + +The -i flag installs the packages that are dependencies of the target. +The -i flag is deprecated. Compiled packages are cached automatically. + +The build flags are shared by the build, clean, get, install, list, run, +and test commands: + + -a + force rebuilding of packages that are already up-to-date. + -n + print the commands but do not run them. + -p n + the number of programs, such as build commands or + test binaries, that can be run in parallel. + The default is the number of CPUs available. + -race + enable data race detection. + Supported only on linux/amd64, freebsd/amd64, darwin/amd64, windows/amd64, + linux/ppc64le and linux/arm64 (only for 48-bit VMA). + -msan + enable interoperation with memory sanitizer. + Supported only on linux/amd64, linux/arm64 + and only with Clang/LLVM as the host C compiler. + On linux/arm64, pie build mode will be used. + -v + print the names of packages as they are compiled. + -work + print the name of the temporary work directory and + do not delete it when exiting. + -x + print the commands. + + -asmflags '[pattern=]arg list' + arguments to pass on each go tool asm invocation. + -buildmode mode + build mode to use. See 'go help buildmode' for more. + -compiler name + name of compiler to use, as in runtime.Compiler (gccgo or gc). + -gccgoflags '[pattern=]arg list' + arguments to pass on each gccgo compiler/linker invocation. + -gcflags '[pattern=]arg list' + arguments to pass on each go tool compile invocation. + -installsuffix suffix + a suffix to use in the name of the package installation directory, + in order to keep output separate from default builds. + If using the -race flag, the install suffix is automatically set to race + or, if set explicitly, has _race appended to it. Likewise for the -msan + flag. Using a -buildmode option that requires non-default compile flags + has a similar effect. + -ldflags '[pattern=]arg list' + arguments to pass on each go tool link invocation. + -linkshared + build code that will be linked against shared libraries previously + created with -buildmode=shared. + -mod mode + module download mode to use: readonly, vendor, or mod. + By default, if a vendor directory is present and the go version in go.mod + is 1.14 or higher, the go command acts as if -mod=vendor were set. + Otherwise, the go command acts as if -mod=readonly were set. + See https://golang.org/ref/mod#build-commands for details. + -modcacherw + leave newly-created directories in the module cache read-write + instead of making them read-only. + -modfile file + in module aware mode, read (and possibly write) an alternate go.mod + file instead of the one in the module root directory. A file named + "go.mod" must still be present in order to determine the module root + directory, but it is not accessed. When -modfile is specified, an + alternate go.sum file is also used: its path is derived from the + -modfile flag by trimming the ".mod" extension and appending ".sum". + -overlay file + read a JSON config file that provides an overlay for build operations. + The file is a JSON struct with a single field, named 'Replace', that + maps each disk file path (a string) to its backing file path, so that + a build will run as if the disk file path exists with the contents + given by the backing file paths, or as if the disk file path does not + exist if its backing file path is empty. Support for the -overlay flag + has some limitations:importantly, cgo files included from outside the + include path must be in the same directory as the Go package they are + included from, and overlays will not appear when binaries and tests are + run through go run and go test respectively. + -pkgdir dir + install and load all packages from dir instead of the usual locations. + For example, when building with a non-standard configuration, + use -pkgdir to keep generated packages in a separate location. + -tags tag,list + a comma-separated list of build tags to consider satisfied during the + build. For more information about build tags, see the description of + build constraints in the documentation for the go/build package. + (Earlier versions of Go used a space-separated list, and that form + is deprecated but still recognized.) + -trimpath + remove all file system paths from the resulting executable. + Instead of absolute file system paths, the recorded file names + will begin with either "go" (for the standard library), + or a module path@version (when using modules), + or a plain import path (when using GOPATH). + -toolexec 'cmd args' + a program to use to invoke toolchain programs like vet and asm. + For example, instead of running asm, the go command will run + 'cmd args /path/to/asm '. + +The -asmflags, -gccgoflags, -gcflags, and -ldflags flags accept a +space-separated list of arguments to pass to an underlying tool +during the build. To embed spaces in an element in the list, surround +it with either single or double quotes. The argument list may be +preceded by a package pattern and an equal sign, which restricts +the use of that argument list to the building of packages matching +that pattern (see 'go help packages' for a description of package +patterns). Without a pattern, the argument list applies only to the +packages named on the command line. The flags may be repeated +with different patterns in order to specify different arguments for +different sets of packages. If a package matches patterns given in +multiple flags, the latest match on the command line wins. +For example, 'go build -gcflags=-S fmt' prints the disassembly +only for package fmt, while 'go build -gcflags=all=-S fmt' +prints the disassembly for fmt and all its dependencies. + +For more about specifying packages, see 'go help packages'. +For more about where packages and binaries are installed, +run 'go help gopath'. +For more about calling between Go and C/C++, run 'go help c'. + +Note: Build adheres to certain conventions such as those described +by 'go help gopath'. Not all projects can follow these conventions, +however. Installations that have their own conventions or that use +a separate software build system may choose to use lower-level +invocations such as 'go tool compile' and 'go tool link' to avoid +some of the overheads and design decisions of the build tool. + +See also: go install, go get, go clean. + `, +} + +const concurrentGCBackendCompilationEnabledByDefault = true + +func init() { + // break init cycle + CmdBuild.Run = runBuild + CmdInstall.Run = runInstall + + CmdBuild.Flag.BoolVar(&cfg.BuildI, "i", false, "") + CmdBuild.Flag.StringVar(&cfg.BuildO, "o", "", "output file or directory") + + CmdInstall.Flag.BoolVar(&cfg.BuildI, "i", false, "") + + AddBuildFlags(CmdBuild, DefaultBuildFlags) + AddBuildFlags(CmdInstall, DefaultBuildFlags) +} + +// Note that flags consulted by other parts of the code +// (for example, buildV) are in cmd/go/internal/cfg. + +var ( + forcedAsmflags []string // internally-forced flags for cmd/asm + forcedGcflags []string // internally-forced flags for cmd/compile + forcedLdflags []string // internally-forced flags for cmd/link + forcedGccgoflags []string // internally-forced flags for gccgo +) + +var BuildToolchain toolchain = noToolchain{} +var ldBuildmode string + +// buildCompiler implements flag.Var. +// It implements Set by updating both +// BuildToolchain and buildContext.Compiler. +type buildCompiler struct{} + +func (c buildCompiler) Set(value string) error { + switch value { + case "gc": + BuildToolchain = gcToolchain{} + case "gccgo": + BuildToolchain = gccgoToolchain{} + default: + return fmt.Errorf("unknown compiler %q", value) + } + cfg.BuildToolchainName = value + cfg.BuildToolchainCompiler = BuildToolchain.compiler + cfg.BuildToolchainLinker = BuildToolchain.linker + cfg.BuildContext.Compiler = value + return nil +} + +func (c buildCompiler) String() string { + return cfg.BuildContext.Compiler +} + +func init() { + switch build.Default.Compiler { + case "gc", "gccgo": + buildCompiler{}.Set(build.Default.Compiler) + } +} + +type BuildFlagMask int + +const ( + DefaultBuildFlags BuildFlagMask = 0 + OmitModFlag BuildFlagMask = 1 << iota + OmitModCommonFlags + OmitVFlag +) + +// AddBuildFlags adds the flags common to the build, clean, get, +// install, list, run, and test commands. +func AddBuildFlags(cmd *base.Command, mask BuildFlagMask) { + base.AddBuildFlagsNX(&cmd.Flag) + cmd.Flag.BoolVar(&cfg.BuildA, "a", false, "") + cmd.Flag.IntVar(&cfg.BuildP, "p", cfg.BuildP, "") + if mask&OmitVFlag == 0 { + cmd.Flag.BoolVar(&cfg.BuildV, "v", false, "") + } + + cmd.Flag.Var(&load.BuildAsmflags, "asmflags", "") + cmd.Flag.Var(buildCompiler{}, "compiler", "") + cmd.Flag.StringVar(&cfg.BuildBuildmode, "buildmode", "default", "") + cmd.Flag.Var(&load.BuildGcflags, "gcflags", "") + cmd.Flag.Var(&load.BuildGccgoflags, "gccgoflags", "") + if mask&OmitModFlag == 0 { + base.AddModFlag(&cmd.Flag) + } + if mask&OmitModCommonFlags == 0 { + base.AddModCommonFlags(&cmd.Flag) + } else { + // Add the overlay flag even when we don't add the rest of the mod common flags. + // This only affects 'go get' in GOPATH mode, but add the flag anyway for + // consistency. + cmd.Flag.StringVar(&fsys.OverlayFile, "overlay", "", "") + } + cmd.Flag.StringVar(&cfg.BuildContext.InstallSuffix, "installsuffix", "", "") + cmd.Flag.Var(&load.BuildLdflags, "ldflags", "") + cmd.Flag.BoolVar(&cfg.BuildLinkshared, "linkshared", false, "") + cmd.Flag.StringVar(&cfg.BuildPkgdir, "pkgdir", "", "") + cmd.Flag.BoolVar(&cfg.BuildRace, "race", false, "") + cmd.Flag.BoolVar(&cfg.BuildMSan, "msan", false, "") + cmd.Flag.Var((*tagsFlag)(&cfg.BuildContext.BuildTags), "tags", "") + cmd.Flag.Var((*base.StringsFlag)(&cfg.BuildToolexec), "toolexec", "") + cmd.Flag.BoolVar(&cfg.BuildTrimpath, "trimpath", false, "") + cmd.Flag.BoolVar(&cfg.BuildWork, "work", false, "") + + // Undocumented, unstable debugging flags. + cmd.Flag.StringVar(&cfg.DebugActiongraph, "debug-actiongraph", "", "") + cmd.Flag.StringVar(&cfg.DebugTrace, "debug-trace", "", "") +} + +// tagsFlag is the implementation of the -tags flag. +type tagsFlag []string + +func (v *tagsFlag) Set(s string) error { + // For compatibility with Go 1.12 and earlier, allow "-tags='a b c'" or even just "-tags='a'". + if strings.Contains(s, " ") || strings.Contains(s, "'") { + return (*base.StringsFlag)(v).Set(s) + } + + // Split on commas, ignore empty strings. + *v = []string{} + for _, s := range strings.Split(s, ",") { + if s != "" { + *v = append(*v, s) + } + } + return nil +} + +func (v *tagsFlag) String() string { + return "" +} + +// fileExtSplit expects a filename and returns the name +// and ext (without the dot). If the file has no +// extension, ext will be empty. +func fileExtSplit(file string) (name, ext string) { + dotExt := filepath.Ext(file) + name = file[:len(file)-len(dotExt)] + if dotExt != "" { + ext = dotExt[1:] + } + return +} + +func pkgsMain(pkgs []*load.Package) (res []*load.Package) { + for _, p := range pkgs { + if p.Name == "main" { + res = append(res, p) + } + } + return res +} + +func pkgsNotMain(pkgs []*load.Package) (res []*load.Package) { + for _, p := range pkgs { + if p.Name != "main" { + res = append(res, p) + } + } + return res +} + +func oneMainPkg(pkgs []*load.Package) []*load.Package { + if len(pkgs) != 1 || pkgs[0].Name != "main" { + base.Fatalf("-buildmode=%s requires exactly one main package", cfg.BuildBuildmode) + } + return pkgs +} + +var pkgsFilter = func(pkgs []*load.Package) []*load.Package { return pkgs } + +var runtimeVersion = runtime.Version() + +func runBuild(ctx context.Context, cmd *base.Command, args []string) { + BuildInit() + var b Builder + b.Init() + + pkgs := load.PackagesAndErrors(ctx, args) + load.CheckPackageErrors(pkgs) + + explicitO := len(cfg.BuildO) > 0 + + if len(pkgs) == 1 && pkgs[0].Name == "main" && cfg.BuildO == "" { + cfg.BuildO = pkgs[0].DefaultExecName() + cfg.BuildO += cfg.ExeSuffix + } + + // sanity check some often mis-used options + switch cfg.BuildContext.Compiler { + case "gccgo": + if load.BuildGcflags.Present() { + fmt.Println("go build: when using gccgo toolchain, please pass compiler flags using -gccgoflags, not -gcflags") + } + if load.BuildLdflags.Present() { + fmt.Println("go build: when using gccgo toolchain, please pass linker flags using -gccgoflags, not -ldflags") + } + case "gc": + if load.BuildGccgoflags.Present() { + fmt.Println("go build: when using gc toolchain, please pass compile flags using -gcflags, and linker flags using -ldflags") + } + } + + depMode := ModeBuild + if cfg.BuildI { + depMode = ModeInstall + fmt.Fprint(os.Stderr, "go build: -i flag is deprecated\n") + } + + pkgs = omitTestOnly(pkgsFilter(pkgs)) + + // Special case -o /dev/null by not writing at all. + if cfg.BuildO == os.DevNull { + cfg.BuildO = "" + } + + if cfg.BuildO != "" { + // If the -o name exists and is a directory or + // ends with a slash or backslash, then + // write all main packages to that directory. + // Otherwise require only a single package be built. + if fi, err := os.Stat(cfg.BuildO); (err == nil && fi.IsDir()) || + strings.HasSuffix(cfg.BuildO, "/") || + strings.HasSuffix(cfg.BuildO, string(os.PathSeparator)) { + if !explicitO { + base.Fatalf("go build: build output %q already exists and is a directory", cfg.BuildO) + } + a := &Action{Mode: "go build"} + for _, p := range pkgs { + if p.Name != "main" { + continue + } + + p.Target = filepath.Join(cfg.BuildO, p.DefaultExecName()) + p.Target += cfg.ExeSuffix + p.Stale = true + p.StaleReason = "build -o flag in use" + a.Deps = append(a.Deps, b.AutoAction(ModeInstall, depMode, p)) + } + if len(a.Deps) == 0 { + base.Fatalf("go build: no main packages to build") + } + b.Do(ctx, a) + return + } + if len(pkgs) > 1 { + base.Fatalf("go build: cannot write multiple packages to non-directory %s", cfg.BuildO) + } else if len(pkgs) == 0 { + base.Fatalf("no packages to build") + } + p := pkgs[0] + p.Target = cfg.BuildO + p.Stale = true // must build - not up to date + p.StaleReason = "build -o flag in use" + a := b.AutoAction(ModeInstall, depMode, p) + b.Do(ctx, a) + return + } + + a := &Action{Mode: "go build"} + for _, p := range pkgs { + a.Deps = append(a.Deps, b.AutoAction(ModeBuild, depMode, p)) + } + if cfg.BuildBuildmode == "shared" { + a = b.buildmodeShared(ModeBuild, depMode, args, pkgs, a) + } + b.Do(ctx, a) +} + +var CmdInstall = &base.Command{ + UsageLine: "go install [build flags] [packages]", + Short: "compile and install packages and dependencies", + Long: ` +Install compiles and installs the packages named by the import paths. + +Executables are installed in the directory named by the GOBIN environment +variable, which defaults to $GOPATH/bin or $HOME/go/bin if the GOPATH +environment variable is not set. Executables in $GOROOT +are installed in $GOROOT/bin or $GOTOOLDIR instead of $GOBIN. + +If the arguments have version suffixes (like @latest or @v1.0.0), "go install" +builds packages in module-aware mode, ignoring the go.mod file in the current +directory or any parent directory, if there is one. This is useful for +installing executables without affecting the dependencies of the main module. +To eliminate ambiguity about which module versions are used in the build, the +arguments must satisfy the following constraints: + +- Arguments must be package paths or package patterns (with "..." wildcards). +They must not be standard packages (like fmt), meta-patterns (std, cmd, +all), or relative or absolute file paths. + +- All arguments must have the same version suffix. Different queries are not +allowed, even if they refer to the same version. + +- All arguments must refer to packages in the same module at the same version. + +- No module is considered the "main" module. If the module containing +packages named on the command line has a go.mod file, it must not contain +directives (replace and exclude) that would cause it to be interpreted +differently than if it were the main module. The module must not require +a higher version of itself. + +- Package path arguments must refer to main packages. Pattern arguments +will only match main packages. + +If the arguments don't have version suffixes, "go install" may run in +module-aware mode or GOPATH mode, depending on the GO111MODULE environment +variable and the presence of a go.mod file. See 'go help modules' for details. +If module-aware mode is enabled, "go install" runs in the context of the main +module. + +When module-aware mode is disabled, other packages are installed in the +directory $GOPATH/pkg/$GOOS_$GOARCH. When module-aware mode is enabled, +other packages are built and cached but not installed. + +The -i flag installs the dependencies of the named packages as well. +The -i flag is deprecated. Compiled packages are cached automatically. + +For more about the build flags, see 'go help build'. +For more about specifying packages, see 'go help packages'. + +See also: go build, go get, go clean. + `, +} + +// libname returns the filename to use for the shared library when using +// -buildmode=shared. The rules we use are: +// Use arguments for special 'meta' packages: +// std --> libstd.so +// std cmd --> libstd,cmd.so +// A single non-meta argument with trailing "/..." is special cased: +// foo/... --> libfoo.so +// (A relative path like "./..." expands the "." first) +// Use import paths for other cases, changing '/' to '-': +// somelib --> libsubdir-somelib.so +// ./ or ../ --> libsubdir-somelib.so +// gopkg.in/tomb.v2 -> libgopkg.in-tomb.v2.so +// a/... b/... ---> liba/c,b/d.so - all matching import paths +// Name parts are joined with ','. +func libname(args []string, pkgs []*load.Package) (string, error) { + var libname string + appendName := func(arg string) { + if libname == "" { + libname = arg + } else { + libname += "," + arg + } + } + var haveNonMeta bool + for _, arg := range args { + if search.IsMetaPackage(arg) { + appendName(arg) + } else { + haveNonMeta = true + } + } + if len(libname) == 0 { // non-meta packages only. use import paths + if len(args) == 1 && strings.HasSuffix(args[0], "/...") { + // Special case of "foo/..." as mentioned above. + arg := strings.TrimSuffix(args[0], "/...") + if build.IsLocalImport(arg) { + cwd, _ := os.Getwd() + bp, _ := cfg.BuildContext.ImportDir(filepath.Join(cwd, arg), build.FindOnly) + if bp.ImportPath != "" && bp.ImportPath != "." { + arg = bp.ImportPath + } + } + appendName(strings.ReplaceAll(arg, "/", "-")) + } else { + for _, pkg := range pkgs { + appendName(strings.ReplaceAll(pkg.ImportPath, "/", "-")) + } + } + } else if haveNonMeta { // have both meta package and a non-meta one + return "", errors.New("mixing of meta and non-meta packages is not allowed") + } + // TODO(mwhudson): Needs to change for platforms that use different naming + // conventions... + return "lib" + libname + ".so", nil +} + +func runInstall(ctx context.Context, cmd *base.Command, args []string) { + // TODO(golang.org/issue/41696): print a deprecation message for the -i flag + // whenever it's set (or just remove it). For now, we don't print a message + // if all named packages are in GOROOT. cmd/dist (run by make.bash) uses + // 'go install -i' when bootstrapping, and we don't want to show deprecation + // messages in that case. + for _, arg := range args { + if strings.Contains(arg, "@") && !build.IsLocalImport(arg) && !filepath.IsAbs(arg) { + if cfg.BuildI { + fmt.Fprint(os.Stderr, "go install: -i flag is deprecated\n") + } + installOutsideModule(ctx, args) + return + } + } + + BuildInit() + pkgs := load.PackagesAndErrors(ctx, args) + if cfg.ModulesEnabled && !modload.HasModRoot() { + haveErrors := false + allMissingErrors := true + for _, pkg := range pkgs { + if pkg.Error == nil { + continue + } + haveErrors = true + if missingErr := (*modload.ImportMissingError)(nil); !errors.As(pkg.Error, &missingErr) { + allMissingErrors = false + break + } + } + if haveErrors && allMissingErrors { + latestArgs := make([]string, len(args)) + for i := range args { + latestArgs[i] = args[i] + "@latest" + } + hint := strings.Join(latestArgs, " ") + base.Fatalf("go install: version is required when current directory is not in a module\n\tTry 'go install %s' to install the latest version", hint) + } + } + load.CheckPackageErrors(pkgs) + if cfg.BuildI { + allGoroot := true + for _, pkg := range pkgs { + if !pkg.Goroot { + allGoroot = false + break + } + } + if !allGoroot { + fmt.Fprint(os.Stderr, "go install: -i flag is deprecated\n") + } + } + + InstallPackages(ctx, args, pkgs) +} + +// omitTestOnly returns pkgs with test-only packages removed. +func omitTestOnly(pkgs []*load.Package) []*load.Package { + var list []*load.Package + for _, p := range pkgs { + if len(p.GoFiles)+len(p.CgoFiles) == 0 && !p.Internal.CmdlinePkgLiteral { + // Package has no source files, + // perhaps due to build tags or perhaps due to only having *_test.go files. + // Also, it is only being processed as the result of a wildcard match + // like ./..., not because it was listed as a literal path on the command line. + // Ignore it. + continue + } + list = append(list, p) + } + return list +} + +func InstallPackages(ctx context.Context, patterns []string, pkgs []*load.Package) { + ctx, span := trace.StartSpan(ctx, "InstallPackages "+strings.Join(patterns, " ")) + defer span.Done() + + if cfg.GOBIN != "" && !filepath.IsAbs(cfg.GOBIN) { + base.Fatalf("cannot install, GOBIN must be an absolute path") + } + + pkgs = omitTestOnly(pkgsFilter(pkgs)) + for _, p := range pkgs { + if p.Target == "" { + switch { + case p.Standard && p.ImportPath == "unsafe": + // unsafe is a built-in package, has no target + case p.Name != "main" && p.Internal.Local && p.ConflictDir == "": + // Non-executables outside GOPATH need not have a target: + // we can use the cache to hold the built package archive for use in future builds. + // The ones inside GOPATH should have a target (in GOPATH/pkg) + // or else something is wrong and worth reporting (like a ConflictDir). + case p.Name != "main" && p.Module != nil: + // Non-executables have no target (except the cache) when building with modules. + case p.Internal.GobinSubdir: + base.Errorf("go %s: cannot install cross-compiled binaries when GOBIN is set", cfg.CmdName) + case p.Internal.CmdlineFiles: + base.Errorf("go %s: no install location for .go files listed on command line (GOBIN not set)", cfg.CmdName) + case p.ConflictDir != "": + base.Errorf("go %s: no install location for %s: hidden by %s", cfg.CmdName, p.Dir, p.ConflictDir) + default: + base.Errorf("go %s: no install location for directory %s outside GOPATH\n"+ + "\tFor more details see: 'go help gopath'", cfg.CmdName, p.Dir) + } + } + } + base.ExitIfErrors() + + var b Builder + b.Init() + depMode := ModeBuild + if cfg.BuildI { + depMode = ModeInstall + } + a := &Action{Mode: "go install"} + var tools []*Action + for _, p := range pkgs { + // If p is a tool, delay the installation until the end of the build. + // This avoids installing assemblers/compilers that are being executed + // by other steps in the build. + a1 := b.AutoAction(ModeInstall, depMode, p) + if load.InstallTargetDir(p) == load.ToTool { + a.Deps = append(a.Deps, a1.Deps...) + a1.Deps = append(a1.Deps, a) + tools = append(tools, a1) + continue + } + a.Deps = append(a.Deps, a1) + } + if len(tools) > 0 { + a = &Action{ + Mode: "go install (tools)", + Deps: tools, + } + } + + if cfg.BuildBuildmode == "shared" { + // Note: If buildmode=shared then only non-main packages + // are present in the pkgs list, so all the special case code about + // tools above did not apply, and a is just a simple Action + // with a list of Deps, one per package named in pkgs, + // the same as in runBuild. + a = b.buildmodeShared(ModeInstall, ModeInstall, patterns, pkgs, a) + } + + b.Do(ctx, a) + base.ExitIfErrors() + + // Success. If this command is 'go install' with no arguments + // and the current directory (the implicit argument) is a command, + // remove any leftover command binary from a previous 'go build'. + // The binary is installed; it's not needed here anymore. + // And worse it might be a stale copy, which you don't want to find + // instead of the installed one if $PATH contains dot. + // One way to view this behavior is that it is as if 'go install' first + // runs 'go build' and the moves the generated file to the install dir. + // See issue 9645. + if len(patterns) == 0 && len(pkgs) == 1 && pkgs[0].Name == "main" { + // Compute file 'go build' would have created. + // If it exists and is an executable file, remove it. + targ := pkgs[0].DefaultExecName() + targ += cfg.ExeSuffix + if filepath.Join(pkgs[0].Dir, targ) != pkgs[0].Target { // maybe $GOBIN is the current directory + fi, err := os.Stat(targ) + if err == nil { + m := fi.Mode() + if m.IsRegular() { + if m&0111 != 0 || cfg.Goos == "windows" { // windows never sets executable bit + os.Remove(targ) + } + } + } + } + } +} + +// installOutsideModule implements 'go install pkg@version'. It builds and +// installs one or more main packages in module mode while ignoring any go.mod +// in the current directory or parent directories. +// +// See golang.org/issue/40276 for details and rationale. +func installOutsideModule(ctx context.Context, args []string) { + modload.ForceUseModules = true + modload.RootMode = modload.NoRoot + modload.AllowMissingModuleImports() + modload.Init() + + // Check that the arguments satisfy syntactic constraints. + var version string + for _, arg := range args { + if i := strings.Index(arg, "@"); i >= 0 { + version = arg[i+1:] + if version == "" { + base.Fatalf("go install %s: version must not be empty", arg) + } + break + } + } + patterns := make([]string, len(args)) + for i, arg := range args { + if !strings.HasSuffix(arg, "@"+version) { + base.Errorf("go install %s: all arguments must have the same version (@%s)", arg, version) + continue + } + p := arg[:len(arg)-len(version)-1] + switch { + case build.IsLocalImport(p): + base.Errorf("go install %s: argument must be a package path, not a relative path", arg) + case filepath.IsAbs(p): + base.Errorf("go install %s: argument must be a package path, not an absolute path", arg) + case search.IsMetaPackage(p): + base.Errorf("go install %s: argument must be a package path, not a meta-package", arg) + case path.Clean(p) != p: + base.Errorf("go install %s: argument must be a clean package path", arg) + case !strings.Contains(p, "...") && search.IsStandardImportPath(p) && goroot.IsStandardPackage(cfg.GOROOT, cfg.BuildContext.Compiler, p): + base.Errorf("go install %s: argument must not be a package in the standard library", arg) + default: + patterns[i] = p + } + } + base.ExitIfErrors() + BuildInit() + + // Query the module providing the first argument, load its go.mod file, and + // check that it doesn't contain directives that would cause it to be + // interpreted differently if it were the main module. + // + // If multiple modules match the first argument, accept the longest match + // (first result). It's possible this module won't provide packages named by + // later arguments, and other modules would. Let's not try to be too + // magical though. + allowed := modload.CheckAllowed + if modload.IsRevisionQuery(version) { + // Don't check for retractions if a specific revision is requested. + allowed = nil + } + noneSelected := func(path string) (version string) { return "none" } + qrs, err := modload.QueryPackages(ctx, patterns[0], version, noneSelected, allowed) + if err != nil { + base.Fatalf("go install %s: %v", args[0], err) + } + installMod := qrs[0].Mod + data, err := modfetch.GoMod(installMod.Path, installMod.Version) + if err != nil { + base.Fatalf("go install %s: %v", args[0], err) + } + f, err := modfile.Parse("go.mod", data, nil) + if err != nil { + base.Fatalf("go install %s: %s: %v", args[0], installMod, err) + } + directiveFmt := "go install %s: %s\n" + + "\tThe go.mod file for the module providing named packages contains one or\n" + + "\tmore %s directives. It must not contain directives that would cause\n" + + "\tit to be interpreted differently than if it were the main module." + if len(f.Replace) > 0 { + base.Fatalf(directiveFmt, args[0], installMod, "replace") + } + if len(f.Exclude) > 0 { + base.Fatalf(directiveFmt, args[0], installMod, "exclude") + } + + // Since we are in NoRoot mode, the build list initially contains only + // the dummy command-line-arguments module. Add a requirement on the + // module that provides the packages named on the command line. + if err := modload.EditBuildList(ctx, nil, []module.Version{installMod}); err != nil { + base.Fatalf("go install %s: %v", args[0], err) + } + + // Load packages for all arguments. Ignore non-main packages. + // Print a warning if an argument contains "..." and matches no main packages. + // PackagesAndErrors already prints warnings for patterns that don't match any + // packages, so be careful not to double print. + matchers := make([]func(string) bool, len(patterns)) + for i, p := range patterns { + if strings.Contains(p, "...") { + matchers[i] = search.MatchPattern(p) + } + } + + // TODO(golang.org/issue/40276): don't report errors loading non-main packages + // matched by a pattern. + pkgs := load.PackagesAndErrors(ctx, patterns) + load.CheckPackageErrors(pkgs) + mainPkgs := make([]*load.Package, 0, len(pkgs)) + mainCount := make([]int, len(patterns)) + nonMainCount := make([]int, len(patterns)) + for _, pkg := range pkgs { + if pkg.Name == "main" { + mainPkgs = append(mainPkgs, pkg) + for i := range patterns { + if matchers[i] != nil && matchers[i](pkg.ImportPath) { + mainCount[i]++ + } + } + } else { + for i := range patterns { + if matchers[i] == nil && patterns[i] == pkg.ImportPath { + base.Errorf("go install: package %s is not a main package", pkg.ImportPath) + } else if matchers[i] != nil && matchers[i](pkg.ImportPath) { + nonMainCount[i]++ + } + } + } + } + base.ExitIfErrors() + for i, p := range patterns { + if matchers[i] != nil && mainCount[i] == 0 && nonMainCount[i] > 0 { + fmt.Fprintf(os.Stderr, "go: warning: %q matched no main packages\n", p) + } + } + + // Check that named packages are all provided by the same module. + for _, pkg := range mainPkgs { + if pkg.Module == nil { + // Packages in std, cmd, and their vendored dependencies + // don't have this field set. + base.Errorf("go install: package %s not provided by module %s", pkg.ImportPath, installMod) + } else if pkg.Module.Path != installMod.Path || pkg.Module.Version != installMod.Version { + base.Errorf("go install: package %s provided by module %s@%s\n\tAll packages must be provided by the same module (%s).", pkg.ImportPath, pkg.Module.Path, pkg.Module.Version, installMod) + } + } + base.ExitIfErrors() + + // Build and install the packages. + InstallPackages(ctx, patterns, mainPkgs) +} + +// ExecCmd is the command to use to run user binaries. +// Normally it is empty, meaning run the binaries directly. +// If cross-compiling and running on a remote system or +// simulator, it is typically go_GOOS_GOARCH_exec, with +// the target GOOS and GOARCH substituted. +// The -exec flag overrides these defaults. +var ExecCmd []string + +// FindExecCmd derives the value of ExecCmd to use. +// It returns that value and leaves ExecCmd set for direct use. +func FindExecCmd() []string { + if ExecCmd != nil { + return ExecCmd + } + ExecCmd = []string{} // avoid work the second time + if cfg.Goos == runtime.GOOS && cfg.Goarch == runtime.GOARCH { + return ExecCmd + } + path, err := exec.LookPath(fmt.Sprintf("go_%s_%s_exec", cfg.Goos, cfg.Goarch)) + if err == nil { + ExecCmd = []string{path} + } + return ExecCmd +} diff --git a/src/cmd/go/internal/work/build_test.go b/src/cmd/go/internal/work/build_test.go new file mode 100644 index 0000000..eaf2639 --- /dev/null +++ b/src/cmd/go/internal/work/build_test.go @@ -0,0 +1,277 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "bytes" + "fmt" + "io/fs" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/load" +) + +func TestRemoveDevNull(t *testing.T) { + fi, err := os.Lstat(os.DevNull) + if err != nil { + t.Skip(err) + } + if fi.Mode().IsRegular() { + t.Errorf("Lstat(%s).Mode().IsRegular() = true; expected false", os.DevNull) + } + mayberemovefile(os.DevNull) + _, err = os.Lstat(os.DevNull) + if err != nil { + t.Errorf("mayberemovefile(%s) did remove it; oops", os.DevNull) + } +} + +func TestSplitPkgConfigOutput(t *testing.T) { + for _, test := range []struct { + in []byte + want []string + }{ + {[]byte(`-r:foo -L/usr/white\ space/lib -lfoo\ bar -lbar\ baz`), []string{"-r:foo", "-L/usr/white space/lib", "-lfoo bar", "-lbar baz"}}, + {[]byte(`-lextra\ fun\ arg\\`), []string{`-lextra fun arg\`}}, + {[]byte("\textra whitespace\r\n"), []string{"extra", "whitespace"}}, + {[]byte(" \r\n "), nil}, + {[]byte(`"-r:foo" "-L/usr/white space/lib" "-lfoo bar" "-lbar baz"`), []string{"-r:foo", "-L/usr/white space/lib", "-lfoo bar", "-lbar baz"}}, + {[]byte(`"-lextra fun arg\\"`), []string{`-lextra fun arg\`}}, + {[]byte(`" \r\n\ "`), []string{` \r\n\ `}}, + {[]byte(`""`), nil}, + {[]byte(``), nil}, + {[]byte(`"\\"`), []string{`\`}}, + {[]byte(`"\x"`), []string{`\x`}}, + {[]byte(`"\\x"`), []string{`\x`}}, + {[]byte(`'\\'`), []string{`\`}}, + {[]byte(`'\x'`), []string{`\x`}}, + {[]byte(`"\\x"`), []string{`\x`}}, + {[]byte(`-fPIC -I/test/include/foo -DQUOTED='"/test/share/doc"'`), []string{"-fPIC", "-I/test/include/foo", `-DQUOTED="/test/share/doc"`}}, + {[]byte(`-fPIC -I/test/include/foo -DQUOTED="/test/share/doc"`), []string{"-fPIC", "-I/test/include/foo", "-DQUOTED=/test/share/doc"}}, + {[]byte(`-fPIC -I/test/include/foo -DQUOTED=\"/test/share/doc\"`), []string{"-fPIC", "-I/test/include/foo", `-DQUOTED="/test/share/doc"`}}, + {[]byte(`-fPIC -I/test/include/foo -DQUOTED='/test/share/doc'`), []string{"-fPIC", "-I/test/include/foo", "-DQUOTED=/test/share/doc"}}, + {[]byte(`-DQUOTED='/te\st/share/d\oc'`), []string{`-DQUOTED=/te\st/share/d\oc`}}, + {[]byte(`-Dhello=10 -Dworld=+32 -DDEFINED_FROM_PKG_CONFIG=hello\ world`), []string{"-Dhello=10", "-Dworld=+32", "-DDEFINED_FROM_PKG_CONFIG=hello world"}}, + {[]byte(`"broken\"" \\\a "a"`), []string{"broken\"", "\\a", "a"}}, + } { + got, err := splitPkgConfigOutput(test.in) + if err != nil { + t.Errorf("splitPkgConfigOutput on %v failed with error %v", test.in, err) + continue + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("splitPkgConfigOutput(%v) = %v; want %v", test.in, got, test.want) + } + } + + for _, test := range []struct { + in []byte + want []string + }{ + // broken quotation + {[]byte(`" \r\n `), nil}, + {[]byte(`"-r:foo" "-L/usr/white space/lib "-lfoo bar" "-lbar baz"`), nil}, + {[]byte(`"-lextra fun arg\\`), nil}, + // broken char escaping + {[]byte(`broken flag\`), nil}, + {[]byte(`extra broken flag \`), nil}, + {[]byte(`\`), nil}, + {[]byte(`"broken\"" "extra" \`), nil}, + } { + got, err := splitPkgConfigOutput(test.in) + if err == nil { + t.Errorf("splitPkgConfigOutput(%v) = %v; haven't failed with error as expected.", test.in, got) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("splitPkgConfigOutput(%v) = %v; want %v", test.in, got, test.want) + } + } + +} + +func TestSharedLibName(t *testing.T) { + // TODO(avdva) - make these values platform-specific + prefix := "lib" + suffix := ".so" + testData := []struct { + args []string + pkgs []*load.Package + expected string + expectErr bool + rootedAt string + }{ + { + args: []string{"std"}, + pkgs: []*load.Package{}, + expected: "std", + }, + { + args: []string{"std", "cmd"}, + pkgs: []*load.Package{}, + expected: "std,cmd", + }, + { + args: []string{}, + pkgs: []*load.Package{pkgImportPath("gopkg.in/somelib")}, + expected: "gopkg.in-somelib", + }, + { + args: []string{"./..."}, + pkgs: []*load.Package{pkgImportPath("somelib")}, + expected: "somelib", + rootedAt: "somelib", + }, + { + args: []string{"../somelib", "../somelib"}, + pkgs: []*load.Package{pkgImportPath("somelib")}, + expected: "somelib", + }, + { + args: []string{"../lib1", "../lib2"}, + pkgs: []*load.Package{pkgImportPath("gopkg.in/lib1"), pkgImportPath("gopkg.in/lib2")}, + expected: "gopkg.in-lib1,gopkg.in-lib2", + }, + { + args: []string{"./..."}, + pkgs: []*load.Package{ + pkgImportPath("gopkg.in/dir/lib1"), + pkgImportPath("gopkg.in/lib2"), + pkgImportPath("gopkg.in/lib3"), + }, + expected: "gopkg.in", + rootedAt: "gopkg.in", + }, + { + args: []string{"std", "../lib2"}, + pkgs: []*load.Package{}, + expectErr: true, + }, + { + args: []string{"all", "./"}, + pkgs: []*load.Package{}, + expectErr: true, + }, + { + args: []string{"cmd", "fmt"}, + pkgs: []*load.Package{}, + expectErr: true, + }, + } + for _, data := range testData { + func() { + if data.rootedAt != "" { + tmpGopath, err := os.MkdirTemp("", "gopath") + if err != nil { + t.Fatal(err) + } + oldGopath := cfg.BuildContext.GOPATH + defer func() { + cfg.BuildContext.GOPATH = oldGopath + os.Chdir(base.Cwd) + err := os.RemoveAll(tmpGopath) + if err != nil { + t.Error(err) + } + }() + root := filepath.Join(tmpGopath, "src", data.rootedAt) + err = os.MkdirAll(root, 0755) + if err != nil { + t.Fatal(err) + } + cfg.BuildContext.GOPATH = tmpGopath + os.Chdir(root) + } + computed, err := libname(data.args, data.pkgs) + if err != nil { + if !data.expectErr { + t.Errorf("libname returned an error %q, expected a name", err.Error()) + } + } else if data.expectErr { + t.Errorf("libname returned %q, expected an error", computed) + } else { + expected := prefix + data.expected + suffix + if expected != computed { + t.Errorf("libname returned %q, expected %q", computed, expected) + } + } + }() + } +} + +func pkgImportPath(pkgpath string) *load.Package { + return &load.Package{ + PackagePublic: load.PackagePublic{ + ImportPath: pkgpath, + }, + } +} + +// When installing packages, the installed package directory should +// respect the SetGID bit and group name of the destination +// directory. +// See https://golang.org/issue/18878. +func TestRespectSetgidDir(t *testing.T) { + switch runtime.GOOS { + case "ios": + t.Skip("can't set SetGID bit with chmod on iOS") + case "windows", "plan9": + t.Skip("chown/chmod setgid are not supported on Windows or Plan 9") + } + + var b Builder + + // Check that `cp` is called instead of `mv` by looking at the output + // of `(*Builder).ShowCmd` afterwards as a sanity check. + cfg.BuildX = true + var cmdBuf bytes.Buffer + b.Print = func(a ...interface{}) (int, error) { + return cmdBuf.WriteString(fmt.Sprint(a...)) + } + + setgiddir, err := os.MkdirTemp("", "SetGroupID") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(setgiddir) + + // BSD mkdir(2) inherits the parent directory group, and other platforms + // can inherit the parent directory group via setgid. The test setup (chmod + // setgid) will fail if the process does not have the group permission to + // the new temporary directory. + err = os.Chown(setgiddir, os.Getuid(), os.Getgid()) + if err != nil { + t.Fatal(err) + } + + // Change setgiddir's permissions to include the SetGID bit. + if err := os.Chmod(setgiddir, 0755|fs.ModeSetgid); err != nil { + t.Fatal(err) + } + + pkgfile, err := os.CreateTemp("", "pkgfile") + if err != nil { + t.Fatalf("os.CreateTemp(\"\", \"pkgfile\"): %v", err) + } + defer os.Remove(pkgfile.Name()) + defer pkgfile.Close() + + dirGIDFile := filepath.Join(setgiddir, "setgid") + if err := b.moveOrCopyFile(dirGIDFile, pkgfile.Name(), 0666, true); err != nil { + t.Fatalf("moveOrCopyFile: %v", err) + } + + got := strings.TrimSpace(cmdBuf.String()) + want := b.fmtcmd("", "cp %s %s", pkgfile.Name(), dirGIDFile) + if got != want { + t.Fatalf("moveOrCopyFile(%q, %q): want %q, got %q", dirGIDFile, pkgfile.Name(), want, got) + } +} diff --git a/src/cmd/go/internal/work/buildid.go b/src/cmd/go/internal/work/buildid.go new file mode 100644 index 0000000..c555d4a --- /dev/null +++ b/src/cmd/go/internal/work/buildid.go @@ -0,0 +1,698 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "bytes" + "fmt" + exec "internal/execabs" + "os" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cache" + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/str" + "cmd/internal/buildid" +) + +// Build IDs +// +// Go packages and binaries are stamped with build IDs that record both +// the action ID, which is a hash of the inputs to the action that produced +// the packages or binary, and the content ID, which is a hash of the action +// output, namely the archive or binary itself. The hash is the same one +// used by the build artifact cache (see cmd/go/internal/cache), but +// truncated when stored in packages and binaries, as the full length is not +// needed and is a bit unwieldy. The precise form is +// +// actionID/[.../]contentID +// +// where the actionID and contentID are prepared by buildid.HashToString below. +// and are found by looking for the first or last slash. +// Usually the buildID is simply actionID/contentID, but see below for an +// exception. +// +// The build ID serves two primary purposes. +// +// 1. The action ID half allows installed packages and binaries to serve as +// one-element cache entries. If we intend to build math.a with a given +// set of inputs summarized in the action ID, and the installed math.a already +// has that action ID, we can reuse the installed math.a instead of rebuilding it. +// +// 2. The content ID half allows the easy preparation of action IDs for steps +// that consume a particular package or binary. The content hash of every +// input file for a given action must be included in the action ID hash. +// Storing the content ID in the build ID lets us read it from the file with +// minimal I/O, instead of reading and hashing the entire file. +// This is especially effective since packages and binaries are typically +// the largest inputs to an action. +// +// Separating action ID from content ID is important for reproducible builds. +// The compiler is compiled with itself. If an output were represented by its +// own action ID (instead of content ID) when computing the action ID of +// the next step in the build process, then the compiler could never have its +// own input action ID as its output action ID (short of a miraculous hash collision). +// Instead we use the content IDs to compute the next action ID, and because +// the content IDs converge, so too do the action IDs and therefore the +// build IDs and the overall compiler binary. See cmd/dist's cmdbootstrap +// for the actual convergence sequence. +// +// The “one-element cache” purpose is a bit more complex for installed +// binaries. For a binary, like cmd/gofmt, there are two steps: compile +// cmd/gofmt/*.go into main.a, and then link main.a into the gofmt binary. +// We do not install gofmt's main.a, only the gofmt binary. Being able to +// decide that the gofmt binary is up-to-date means computing the action ID +// for the final link of the gofmt binary and comparing it against the +// already-installed gofmt binary. But computing the action ID for the link +// means knowing the content ID of main.a, which we did not keep. +// To sidestep this problem, each binary actually stores an expanded build ID: +// +// actionID(binary)/actionID(main.a)/contentID(main.a)/contentID(binary) +// +// (Note that this can be viewed equivalently as: +// +// actionID(binary)/buildID(main.a)/contentID(binary) +// +// Storing the buildID(main.a) in the middle lets the computations that care +// about the prefix or suffix halves ignore the middle and preserves the +// original build ID as a contiguous string.) +// +// During the build, when it's time to build main.a, the gofmt binary has the +// information needed to decide whether the eventual link would produce +// the same binary: if the action ID for main.a's inputs matches and then +// the action ID for the link step matches when assuming the given main.a +// content ID, then the binary as a whole is up-to-date and need not be rebuilt. +// +// This is all a bit complex and may be simplified once we can rely on the +// main cache, but at least at the start we will be using the content-based +// staleness determination without a cache beyond the usual installed +// package and binary locations. + +const buildIDSeparator = "/" + +// actionID returns the action ID half of a build ID. +func actionID(buildID string) string { + i := strings.Index(buildID, buildIDSeparator) + if i < 0 { + return buildID + } + return buildID[:i] +} + +// contentID returns the content ID half of a build ID. +func contentID(buildID string) string { + return buildID[strings.LastIndex(buildID, buildIDSeparator)+1:] +} + +// toolID returns the unique ID to use for the current copy of the +// named tool (asm, compile, cover, link). +// +// It is important that if the tool changes (for example a compiler bug is fixed +// and the compiler reinstalled), toolID returns a different string, so that old +// package archives look stale and are rebuilt (with the fixed compiler). +// This suggests using a content hash of the tool binary, as stored in the build ID. +// +// Unfortunately, we can't just open the tool binary, because the tool might be +// invoked via a wrapper program specified by -toolexec and we don't know +// what the wrapper program does. In particular, we want "-toolexec toolstash" +// to continue working: it does no good if "-toolexec toolstash" is executing a +// stashed copy of the compiler but the go command is acting as if it will run +// the standard copy of the compiler. The solution is to ask the tool binary to tell +// us its own build ID using the "-V=full" flag now supported by all tools. +// Then we know we're getting the build ID of the compiler that will actually run +// during the build. (How does the compiler binary know its own content hash? +// We store it there using updateBuildID after the standard link step.) +// +// A final twist is that we'd prefer to have reproducible builds for release toolchains. +// It should be possible to cross-compile for Windows from either Linux or Mac +// or Windows itself and produce the same binaries, bit for bit. If the tool ID, +// which influences the action ID half of the build ID, is based on the content ID, +// then the Linux compiler binary and Mac compiler binary will have different tool IDs +// and therefore produce executables with different action IDs. +// To avoid this problem, for releases we use the release version string instead +// of the compiler binary's content hash. This assumes that all compilers built +// on all different systems are semantically equivalent, which is of course only true +// modulo bugs. (Producing the exact same executables also requires that the different +// build setups agree on details like $GOROOT and file name paths, but at least the +// tool IDs do not make it impossible.) +func (b *Builder) toolID(name string) string { + b.id.Lock() + id := b.toolIDCache[name] + b.id.Unlock() + + if id != "" { + return id + } + + path := base.Tool(name) + desc := "go tool " + name + + // Special case: undocumented -vettool overrides usual vet, + // for testing vet or supplying an alternative analysis tool. + if name == "vet" && VetTool != "" { + path = VetTool + desc = VetTool + } + + cmdline := str.StringList(cfg.BuildToolexec, path, "-V=full") + cmd := exec.Command(cmdline[0], cmdline[1:]...) + cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + base.Fatalf("%s: %v\n%s%s", desc, err, stdout.Bytes(), stderr.Bytes()) + } + + line := stdout.String() + f := strings.Fields(line) + if len(f) < 3 || f[0] != name && path != VetTool || f[1] != "version" || f[2] == "devel" && !strings.HasPrefix(f[len(f)-1], "buildID=") { + base.Fatalf("%s -V=full: unexpected output:\n\t%s", desc, line) + } + if f[2] == "devel" { + // On the development branch, use the content ID part of the build ID. + id = contentID(f[len(f)-1]) + } else { + // For a release, the output is like: "compile version go1.9.1 X:framepointer". + // Use the whole line. + id = strings.TrimSpace(line) + } + + b.id.Lock() + b.toolIDCache[name] = id + b.id.Unlock() + + return id +} + +// gccToolID returns the unique ID to use for a tool that is invoked +// by the GCC driver. This is used particularly for gccgo, but this can also +// be used for gcc, g++, gfortran, etc.; those tools all use the GCC +// driver under different names. The approach used here should also +// work for sufficiently new versions of clang. Unlike toolID, the +// name argument is the program to run. The language argument is the +// type of input file as passed to the GCC driver's -x option. +// +// For these tools we have no -V=full option to dump the build ID, +// but we can run the tool with -v -### to reliably get the compiler proper +// and hash that. That will work in the presence of -toolexec. +// +// In order to get reproducible builds for released compilers, we +// detect a released compiler by the absence of "experimental" in the +// --version output, and in that case we just use the version string. +func (b *Builder) gccgoToolID(name, language string) (string, error) { + key := name + "." + language + b.id.Lock() + id := b.toolIDCache[key] + b.id.Unlock() + + if id != "" { + return id, nil + } + + // Invoke the driver with -### to see the subcommands and the + // version strings. Use -x to set the language. Pretend to + // compile an empty file on standard input. + cmdline := str.StringList(cfg.BuildToolexec, name, "-###", "-x", language, "-c", "-") + cmd := exec.Command(cmdline[0], cmdline[1:]...) + cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir) + // Force untranslated output so that we see the string "version". + cmd.Env = append(cmd.Env, "LC_ALL=C") + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("%s: %v; output: %q", name, err, out) + } + + version := "" + lines := strings.Split(string(out), "\n") + for _, line := range lines { + if fields := strings.Fields(line); len(fields) > 1 && fields[1] == "version" { + version = line + break + } + } + if version == "" { + return "", fmt.Errorf("%s: can not find version number in %q", name, out) + } + + if !strings.Contains(version, "experimental") { + // This is a release. Use this line as the tool ID. + id = version + } else { + // This is a development version. The first line with + // a leading space is the compiler proper. + compiler := "" + for _, line := range lines { + if len(line) > 1 && line[0] == ' ' { + compiler = line + break + } + } + if compiler == "" { + return "", fmt.Errorf("%s: can not find compilation command in %q", name, out) + } + + fields := strings.Fields(compiler) + if len(fields) == 0 { + return "", fmt.Errorf("%s: compilation command confusion %q", name, out) + } + exe := fields[0] + if !strings.ContainsAny(exe, `/\`) { + if lp, err := exec.LookPath(exe); err == nil { + exe = lp + } + } + id, err = buildid.ReadFile(exe) + if err != nil { + return "", err + } + + // If we can't find a build ID, use a hash. + if id == "" { + id = b.fileHash(exe) + } + } + + b.id.Lock() + b.toolIDCache[key] = id + b.id.Unlock() + + return id, nil +} + +// Check if assembler used by gccgo is GNU as. +func assemblerIsGas() bool { + cmd := exec.Command(BuildToolchain.compiler(), "-print-prog-name=as") + assembler, err := cmd.Output() + if err == nil { + cmd := exec.Command(strings.TrimSpace(string(assembler)), "--version") + out, err := cmd.Output() + return err == nil && strings.Contains(string(out), "GNU") + } else { + return false + } +} + +// gccgoBuildIDFile creates an assembler file that records the +// action's build ID in an SHF_EXCLUDE section for ELF files or +// in a CSECT in XCOFF files. +func (b *Builder) gccgoBuildIDFile(a *Action) (string, error) { + sfile := a.Objdir + "_buildid.s" + + var buf bytes.Buffer + if cfg.Goos == "aix" { + fmt.Fprintf(&buf, "\t.csect .go.buildid[XO]\n") + } else if (cfg.Goos != "solaris" && cfg.Goos != "illumos") || assemblerIsGas() { + fmt.Fprintf(&buf, "\t"+`.section .go.buildid,"e"`+"\n") + } else if cfg.Goarch == "sparc" || cfg.Goarch == "sparc64" { + fmt.Fprintf(&buf, "\t"+`.section ".go.buildid",#exclude`+"\n") + } else { // cfg.Goarch == "386" || cfg.Goarch == "amd64" + fmt.Fprintf(&buf, "\t"+`.section .go.buildid,#exclude`+"\n") + } + fmt.Fprintf(&buf, "\t.byte ") + for i := 0; i < len(a.buildID); i++ { + if i > 0 { + if i%8 == 0 { + fmt.Fprintf(&buf, "\n\t.byte ") + } else { + fmt.Fprintf(&buf, ",") + } + } + fmt.Fprintf(&buf, "%#02x", a.buildID[i]) + } + fmt.Fprintf(&buf, "\n") + if cfg.Goos != "solaris" && cfg.Goos != "illumos" && cfg.Goos != "aix" { + secType := "@progbits" + if cfg.Goarch == "arm" { + secType = "%progbits" + } + fmt.Fprintf(&buf, "\t"+`.section .note.GNU-stack,"",%s`+"\n", secType) + fmt.Fprintf(&buf, "\t"+`.section .note.GNU-split-stack,"",%s`+"\n", secType) + } + + if cfg.BuildN || cfg.BuildX { + for _, line := range bytes.Split(buf.Bytes(), []byte("\n")) { + b.Showcmd("", "echo '%s' >> %s", line, sfile) + } + if cfg.BuildN { + return sfile, nil + } + } + + if err := os.WriteFile(sfile, buf.Bytes(), 0666); err != nil { + return "", err + } + + return sfile, nil +} + +// buildID returns the build ID found in the given file. +// If no build ID is found, buildID returns the content hash of the file. +func (b *Builder) buildID(file string) string { + b.id.Lock() + id := b.buildIDCache[file] + b.id.Unlock() + + if id != "" { + return id + } + + id, err := buildid.ReadFile(file) + if err != nil { + id = b.fileHash(file) + } + + b.id.Lock() + b.buildIDCache[file] = id + b.id.Unlock() + + return id +} + +// fileHash returns the content hash of the named file. +func (b *Builder) fileHash(file string) string { + file, _ = fsys.OverlayPath(file) + sum, err := cache.FileHash(file) + if err != nil { + return "" + } + return buildid.HashToString(sum) +} + +// useCache tries to satisfy the action a, which has action ID actionHash, +// by using a cached result from an earlier build. At the moment, the only +// cached result is the installed package or binary at target. +// If useCache decides that the cache can be used, it sets a.buildID +// and a.built for use by parent actions and then returns true. +// Otherwise it sets a.buildID to a temporary build ID for use in the build +// and returns false. When useCache returns false the expectation is that +// the caller will build the target and then call updateBuildID to finish the +// build ID computation. +// When useCache returns false, it may have initiated buffering of output +// during a's work. The caller should defer b.flushOutput(a), to make sure +// that flushOutput is eventually called regardless of whether the action +// succeeds. The flushOutput call must happen after updateBuildID. +func (b *Builder) useCache(a *Action, actionHash cache.ActionID, target string) bool { + // The second half of the build ID here is a placeholder for the content hash. + // It's important that the overall buildID be unlikely verging on impossible + // to appear in the output by chance, but that should be taken care of by + // the actionID half; if it also appeared in the input that would be like an + // engineered 120-bit partial SHA256 collision. + a.actionID = actionHash + actionID := buildid.HashToString(actionHash) + if a.json != nil { + a.json.ActionID = actionID + } + contentID := actionID // temporary placeholder, likely unique + a.buildID = actionID + buildIDSeparator + contentID + + // Executable binaries also record the main build ID in the middle. + // See "Build IDs" comment above. + if a.Mode == "link" { + mainpkg := a.Deps[0] + a.buildID = actionID + buildIDSeparator + mainpkg.buildID + buildIDSeparator + contentID + } + + // Check to see if target exists and matches the expected action ID. + // If so, it's up to date and we can reuse it instead of rebuilding it. + var buildID string + if target != "" && !cfg.BuildA { + buildID, _ = buildid.ReadFile(target) + if strings.HasPrefix(buildID, actionID+buildIDSeparator) { + a.buildID = buildID + if a.json != nil { + a.json.BuildID = a.buildID + } + a.built = target + // Poison a.Target to catch uses later in the build. + a.Target = "DO NOT USE - " + a.Mode + return true + } + } + + // Special case for building a main package: if the only thing we + // want the package for is to link a binary, and the binary is + // already up-to-date, then to avoid a rebuild, report the package + // as up-to-date as well. See "Build IDs" comment above. + // TODO(rsc): Rewrite this code to use a TryCache func on the link action. + if target != "" && !cfg.BuildA && !b.NeedExport && a.Mode == "build" && len(a.triggers) == 1 && a.triggers[0].Mode == "link" { + buildID, err := buildid.ReadFile(target) + if err == nil { + id := strings.Split(buildID, buildIDSeparator) + if len(id) == 4 && id[1] == actionID { + // Temporarily assume a.buildID is the package build ID + // stored in the installed binary, and see if that makes + // the upcoming link action ID a match. If so, report that + // we built the package, safe in the knowledge that the + // link step will not ask us for the actual package file. + // Note that (*Builder).LinkAction arranged that all of + // a.triggers[0]'s dependencies other than a are also + // dependencies of a, so that we can be sure that, + // other than a.buildID, b.linkActionID is only accessing + // build IDs of completed actions. + oldBuildID := a.buildID + a.buildID = id[1] + buildIDSeparator + id[2] + linkID := buildid.HashToString(b.linkActionID(a.triggers[0])) + if id[0] == linkID { + // Best effort attempt to display output from the compile and link steps. + // If it doesn't work, it doesn't work: reusing the cached binary is more + // important than reprinting diagnostic information. + if c := cache.Default(); c != nil { + showStdout(b, c, a.actionID, "stdout") // compile output + showStdout(b, c, a.actionID, "link-stdout") // link output + } + + // Poison a.Target to catch uses later in the build. + a.Target = "DO NOT USE - main build pseudo-cache Target" + a.built = "DO NOT USE - main build pseudo-cache built" + if a.json != nil { + a.json.BuildID = a.buildID + } + return true + } + // Otherwise restore old build ID for main build. + a.buildID = oldBuildID + } + } + } + + // Special case for linking a test binary: if the only thing we + // want the binary for is to run the test, and the test result is cached, + // then to avoid the link step, report the link as up-to-date. + // We avoid the nested build ID problem in the previous special case + // by recording the test results in the cache under the action ID half. + if !cfg.BuildA && len(a.triggers) == 1 && a.triggers[0].TryCache != nil && a.triggers[0].TryCache(b, a.triggers[0]) { + // Best effort attempt to display output from the compile and link steps. + // If it doesn't work, it doesn't work: reusing the test result is more + // important than reprinting diagnostic information. + if c := cache.Default(); c != nil { + showStdout(b, c, a.Deps[0].actionID, "stdout") // compile output + showStdout(b, c, a.Deps[0].actionID, "link-stdout") // link output + } + + // Poison a.Target to catch uses later in the build. + a.Target = "DO NOT USE - pseudo-cache Target" + a.built = "DO NOT USE - pseudo-cache built" + return true + } + + if b.IsCmdList { + // Invoked during go list to compute and record staleness. + if p := a.Package; p != nil && !p.Stale { + p.Stale = true + if cfg.BuildA { + p.StaleReason = "build -a flag in use" + } else { + p.StaleReason = "build ID mismatch" + for _, p1 := range p.Internal.Imports { + if p1.Stale && p1.StaleReason != "" { + if strings.HasPrefix(p1.StaleReason, "stale dependency: ") { + p.StaleReason = p1.StaleReason + break + } + if strings.HasPrefix(p.StaleReason, "build ID mismatch") { + p.StaleReason = "stale dependency: " + p1.ImportPath + } + } + } + } + } + + // Fall through to update a.buildID from the build artifact cache, + // which will affect the computation of buildIDs for targets + // higher up in the dependency graph. + } + + // Check the build artifact cache. + // We treat hits in this cache as being "stale" for the purposes of go list + // (in effect, "stale" means whether p.Target is up-to-date), + // but we're still happy to use results from the build artifact cache. + if c := cache.Default(); c != nil { + if !cfg.BuildA { + if file, _, err := c.GetFile(actionHash); err == nil { + if buildID, err := buildid.ReadFile(file); err == nil { + if err := showStdout(b, c, a.actionID, "stdout"); err == nil { + a.built = file + a.Target = "DO NOT USE - using cache" + a.buildID = buildID + if a.json != nil { + a.json.BuildID = a.buildID + } + if p := a.Package; p != nil { + // Clearer than explaining that something else is stale. + p.StaleReason = "not installed but available in build cache" + } + return true + } + } + } + } + + // Begin saving output for later writing to cache. + a.output = []byte{} + } + + return false +} + +func showStdout(b *Builder, c *cache.Cache, actionID cache.ActionID, key string) error { + stdout, stdoutEntry, err := c.GetBytes(cache.Subkey(actionID, key)) + if err != nil { + return err + } + + if len(stdout) > 0 { + if cfg.BuildX || cfg.BuildN { + b.Showcmd("", "%s # internal", joinUnambiguously(str.StringList("cat", c.OutputFile(stdoutEntry.OutputID)))) + } + if !cfg.BuildN { + b.Print(string(stdout)) + } + } + return nil +} + +// flushOutput flushes the output being queued in a. +func (b *Builder) flushOutput(a *Action) { + b.Print(string(a.output)) + a.output = nil +} + +// updateBuildID updates the build ID in the target written by action a. +// It requires that useCache was called for action a and returned false, +// and that the build was then carried out and given the temporary +// a.buildID to record as the build ID in the resulting package or binary. +// updateBuildID computes the final content ID and updates the build IDs +// in the binary. +// +// Keep in sync with src/cmd/buildid/buildid.go +func (b *Builder) updateBuildID(a *Action, target string, rewrite bool) error { + if cfg.BuildX || cfg.BuildN { + if rewrite { + b.Showcmd("", "%s # internal", joinUnambiguously(str.StringList(base.Tool("buildid"), "-w", target))) + } + if cfg.BuildN { + return nil + } + } + + // Cache output from compile/link, even if we don't do the rest. + if c := cache.Default(); c != nil { + switch a.Mode { + case "build": + c.PutBytes(cache.Subkey(a.actionID, "stdout"), a.output) + case "link": + // Even though we don't cache the binary, cache the linker text output. + // We might notice that an installed binary is up-to-date but still + // want to pretend to have run the linker. + // Store it under the main package's action ID + // to make it easier to find when that's all we have. + for _, a1 := range a.Deps { + if p1 := a1.Package; p1 != nil && p1.Name == "main" { + c.PutBytes(cache.Subkey(a1.actionID, "link-stdout"), a.output) + break + } + } + } + } + + // Find occurrences of old ID and compute new content-based ID. + r, err := os.Open(target) + if err != nil { + return err + } + matches, hash, err := buildid.FindAndHash(r, a.buildID, 0) + r.Close() + if err != nil { + return err + } + newID := a.buildID[:strings.LastIndex(a.buildID, buildIDSeparator)] + buildIDSeparator + buildid.HashToString(hash) + if len(newID) != len(a.buildID) { + return fmt.Errorf("internal error: build ID length mismatch %q vs %q", a.buildID, newID) + } + + // Replace with new content-based ID. + a.buildID = newID + if a.json != nil { + a.json.BuildID = a.buildID + } + if len(matches) == 0 { + // Assume the user specified -buildid= to override what we were going to choose. + return nil + } + + if rewrite { + w, err := os.OpenFile(target, os.O_RDWR, 0) + if err != nil { + return err + } + err = buildid.Rewrite(w, matches, newID) + if err != nil { + w.Close() + return err + } + if err := w.Close(); err != nil { + return err + } + } + + // Cache package builds, but not binaries (link steps). + // The expectation is that binaries are not reused + // nearly as often as individual packages, and they're + // much larger, so the cache-footprint-to-utility ratio + // of binaries is much lower for binaries. + // Not caching the link step also makes sure that repeated "go run" at least + // always rerun the linker, so that they don't get too fast. + // (We don't want people thinking go is a scripting language.) + // Note also that if we start caching binaries, then we will + // copy the binaries out of the cache to run them, and then + // that will mean the go process is itself writing a binary + // and then executing it, so we will need to defend against + // ETXTBSY problems as discussed in exec.go and golang.org/issue/22220. + if c := cache.Default(); c != nil && a.Mode == "build" { + r, err := os.Open(target) + if err == nil { + if a.output == nil { + panic("internal error: a.output not set") + } + outputID, _, err := c.Put(a.actionID, r) + r.Close() + if err == nil && cfg.BuildX { + b.Showcmd("", "%s # internal", joinUnambiguously(str.StringList("cp", target, c.OutputFile(outputID)))) + } + if b.NeedExport { + if err != nil { + return err + } + a.Package.Export = c.OutputFile(outputID) + a.Package.BuildID = a.buildID + } + } + } + + return nil +} diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go new file mode 100644 index 0000000..422e83c --- /dev/null +++ b/src/cmd/go/internal/work/exec.go @@ -0,0 +1,3316 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Action graph execution. + +package work + +import ( + "bytes" + "cmd/go/internal/fsys" + "context" + "encoding/json" + "errors" + "fmt" + exec "internal/execabs" + "internal/lazyregexp" + "io" + "io/fs" + "log" + "math/rand" + "os" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "sync" + "time" + + "cmd/go/internal/base" + "cmd/go/internal/cache" + "cmd/go/internal/cfg" + "cmd/go/internal/load" + "cmd/go/internal/modload" + "cmd/go/internal/str" + "cmd/go/internal/trace" +) + +// actionList returns the list of actions in the dag rooted at root +// as visited in a depth-first post-order traversal. +func actionList(root *Action) []*Action { + seen := map[*Action]bool{} + all := []*Action{} + var walk func(*Action) + walk = func(a *Action) { + if seen[a] { + return + } + seen[a] = true + for _, a1 := range a.Deps { + walk(a1) + } + all = append(all, a) + } + walk(root) + return all +} + +// do runs the action graph rooted at root. +func (b *Builder) Do(ctx context.Context, root *Action) { + ctx, span := trace.StartSpan(ctx, "exec.Builder.Do ("+root.Mode+" "+root.Target+")") + defer span.Done() + + if !b.IsCmdList { + // If we're doing real work, take time at the end to trim the cache. + c := cache.Default() + defer c.Trim() + } + + // Build list of all actions, assigning depth-first post-order priority. + // The original implementation here was a true queue + // (using a channel) but it had the effect of getting + // distracted by low-level leaf actions to the detriment + // of completing higher-level actions. The order of + // work does not matter much to overall execution time, + // but when running "go test std" it is nice to see each test + // results as soon as possible. The priorities assigned + // ensure that, all else being equal, the execution prefers + // to do what it would have done first in a simple depth-first + // dependency order traversal. + all := actionList(root) + for i, a := range all { + a.priority = i + } + + // Write action graph, without timing information, in case we fail and exit early. + writeActionGraph := func() { + if file := cfg.DebugActiongraph; file != "" { + if strings.HasSuffix(file, ".go") { + // Do not overwrite Go source code in: + // go build -debug-actiongraph x.go + base.Fatalf("go: refusing to write action graph to %v\n", file) + } + js := actionGraphJSON(root) + if err := os.WriteFile(file, []byte(js), 0666); err != nil { + fmt.Fprintf(os.Stderr, "go: writing action graph: %v\n", err) + base.SetExitStatus(1) + } + } + } + writeActionGraph() + + b.readySema = make(chan bool, len(all)) + + // Initialize per-action execution state. + for _, a := range all { + for _, a1 := range a.Deps { + a1.triggers = append(a1.triggers, a) + } + a.pending = len(a.Deps) + if a.pending == 0 { + b.ready.push(a) + b.readySema <- true + } + } + + // Handle runs a single action and takes care of triggering + // any actions that are runnable as a result. + handle := func(ctx context.Context, a *Action) { + if a.json != nil { + a.json.TimeStart = time.Now() + } + var err error + if a.Func != nil && (!a.Failed || a.IgnoreFail) { + // TODO(matloob): Better action descriptions + desc := "Executing action " + if a.Package != nil { + desc += "(" + a.Mode + " " + a.Package.Desc() + ")" + } + ctx, span := trace.StartSpan(ctx, desc) + a.traceSpan = span + for _, d := range a.Deps { + trace.Flow(ctx, d.traceSpan, a.traceSpan) + } + err = a.Func(b, ctx, a) + span.Done() + } + if a.json != nil { + a.json.TimeDone = time.Now() + } + + // The actions run in parallel but all the updates to the + // shared work state are serialized through b.exec. + b.exec.Lock() + defer b.exec.Unlock() + + if err != nil { + if err == errPrintedOutput { + base.SetExitStatus(2) + } else { + base.Errorf("%s", err) + } + a.Failed = true + } + + for _, a0 := range a.triggers { + if a.Failed { + a0.Failed = true + } + if a0.pending--; a0.pending == 0 { + b.ready.push(a0) + b.readySema <- true + } + } + + if a == root { + close(b.readySema) + } + } + + var wg sync.WaitGroup + + // Kick off goroutines according to parallelism. + // If we are using the -n flag (just printing commands) + // drop the parallelism to 1, both to make the output + // deterministic and because there is no real work anyway. + par := cfg.BuildP + if cfg.BuildN { + par = 1 + } + for i := 0; i < par; i++ { + wg.Add(1) + go func() { + ctx := trace.StartGoroutine(ctx) + defer wg.Done() + for { + select { + case _, ok := <-b.readySema: + if !ok { + return + } + // Receiving a value from b.readySema entitles + // us to take from the ready queue. + b.exec.Lock() + a := b.ready.pop() + b.exec.Unlock() + handle(ctx, a) + case <-base.Interrupted: + base.SetExitStatus(1) + return + } + } + }() + } + + wg.Wait() + + // Write action graph again, this time with timing information. + writeActionGraph() +} + +// buildActionID computes the action ID for a build action. +func (b *Builder) buildActionID(a *Action) cache.ActionID { + p := a.Package + h := cache.NewHash("build " + p.ImportPath) + + // Configuration independent of compiler toolchain. + // Note: buildmode has already been accounted for in buildGcflags + // and should not be inserted explicitly. Most buildmodes use the + // same compiler settings and can reuse each other's results. + // If not, the reason is already recorded in buildGcflags. + fmt.Fprintf(h, "compile\n") + // Only include the package directory if it may affect the output. + // We trim workspace paths for all packages when -trimpath is set. + // The compiler hides the exact value of $GOROOT + // when building things in GOROOT. + // Assume b.WorkDir is being trimmed properly. + // When -trimpath is used with a package built from the module cache, + // use the module path and version instead of the directory. + if !p.Goroot && !cfg.BuildTrimpath && !strings.HasPrefix(p.Dir, b.WorkDir) { + fmt.Fprintf(h, "dir %s\n", p.Dir) + } else if cfg.BuildTrimpath && p.Module != nil { + fmt.Fprintf(h, "module %s@%s\n", p.Module.Path, p.Module.Version) + } + if p.Module != nil { + fmt.Fprintf(h, "go %s\n", p.Module.GoVersion) + } + fmt.Fprintf(h, "goos %s goarch %s\n", cfg.Goos, cfg.Goarch) + fmt.Fprintf(h, "import %q\n", p.ImportPath) + fmt.Fprintf(h, "omitdebug %v standard %v local %v prefix %q\n", p.Internal.OmitDebug, p.Standard, p.Internal.Local, p.Internal.LocalPrefix) + if cfg.BuildTrimpath { + fmt.Fprintln(h, "trimpath") + } + if p.Internal.ForceLibrary { + fmt.Fprintf(h, "forcelibrary\n") + } + if len(p.CgoFiles)+len(p.SwigFiles) > 0 { + fmt.Fprintf(h, "cgo %q\n", b.toolID("cgo")) + cppflags, cflags, cxxflags, fflags, ldflags, _ := b.CFlags(p) + fmt.Fprintf(h, "CC=%q %q %q %q\n", b.ccExe(), cppflags, cflags, ldflags) + if len(p.CXXFiles)+len(p.SwigFiles) > 0 { + fmt.Fprintf(h, "CXX=%q %q\n", b.cxxExe(), cxxflags) + } + if len(p.FFiles) > 0 { + fmt.Fprintf(h, "FC=%q %q\n", b.fcExe(), fflags) + } + // TODO(rsc): Should we include the SWIG version or Fortran/GCC/G++/Objective-C compiler versions? + } + if p.Internal.CoverMode != "" { + fmt.Fprintf(h, "cover %q %q\n", p.Internal.CoverMode, b.toolID("cover")) + } + fmt.Fprintf(h, "modinfo %q\n", p.Internal.BuildInfo) + + // Configuration specific to compiler toolchain. + switch cfg.BuildToolchainName { + default: + base.Fatalf("buildActionID: unknown build toolchain %q", cfg.BuildToolchainName) + case "gc": + fmt.Fprintf(h, "compile %s %q %q\n", b.toolID("compile"), forcedGcflags, p.Internal.Gcflags) + if len(p.SFiles) > 0 { + fmt.Fprintf(h, "asm %q %q %q\n", b.toolID("asm"), forcedAsmflags, p.Internal.Asmflags) + } + + // GOARM, GOMIPS, etc. + key, val := cfg.GetArchEnv() + fmt.Fprintf(h, "%s=%s\n", key, val) + + // TODO(rsc): Convince compiler team not to add more magic environment variables, + // or perhaps restrict the environment variables passed to subprocesses. + // Because these are clumsy, undocumented special-case hacks + // for debugging the compiler, they are not settable using 'go env -w', + // and so here we use os.Getenv, not cfg.Getenv. + magic := []string{ + "GOCLOBBERDEADHASH", + "GOSSAFUNC", + "GO_SSA_PHI_LOC_CUTOFF", + "GOSSAHASH", + } + for _, env := range magic { + if x := os.Getenv(env); x != "" { + fmt.Fprintf(h, "magic %s=%s\n", env, x) + } + } + if os.Getenv("GOSSAHASH") != "" { + for i := 0; ; i++ { + env := fmt.Sprintf("GOSSAHASH%d", i) + x := os.Getenv(env) + if x == "" { + break + } + fmt.Fprintf(h, "magic %s=%s\n", env, x) + } + } + if os.Getenv("GSHS_LOGFILE") != "" { + // Clumsy hack. Compiler writes to this log file, + // so do not allow use of cache at all. + // We will still write to the cache but it will be + // essentially unfindable. + fmt.Fprintf(h, "nocache %d\n", time.Now().UnixNano()) + } + + case "gccgo": + id, err := b.gccgoToolID(BuildToolchain.compiler(), "go") + if err != nil { + base.Fatalf("%v", err) + } + fmt.Fprintf(h, "compile %s %q %q\n", id, forcedGccgoflags, p.Internal.Gccgoflags) + fmt.Fprintf(h, "pkgpath %s\n", gccgoPkgpath(p)) + fmt.Fprintf(h, "ar %q\n", BuildToolchain.(gccgoToolchain).ar()) + if len(p.SFiles) > 0 { + id, _ = b.gccgoToolID(BuildToolchain.compiler(), "assembler-with-cpp") + // Ignore error; different assembler versions + // are unlikely to make any difference anyhow. + fmt.Fprintf(h, "asm %q\n", id) + } + } + + // Input files. + inputFiles := str.StringList( + p.GoFiles, + p.CgoFiles, + p.CFiles, + p.CXXFiles, + p.FFiles, + p.MFiles, + p.HFiles, + p.SFiles, + p.SysoFiles, + p.SwigFiles, + p.SwigCXXFiles, + p.EmbedFiles, + ) + for _, file := range inputFiles { + fmt.Fprintf(h, "file %s %s\n", file, b.fileHash(filepath.Join(p.Dir, file))) + } + for _, a1 := range a.Deps { + p1 := a1.Package + if p1 != nil { + fmt.Fprintf(h, "import %s %s\n", p1.ImportPath, contentID(a1.buildID)) + } + } + + return h.Sum() +} + +// needCgoHdr reports whether the actions triggered by this one +// expect to be able to access the cgo-generated header file. +func (b *Builder) needCgoHdr(a *Action) bool { + // If this build triggers a header install, run cgo to get the header. + if !b.IsCmdList && (a.Package.UsesCgo() || a.Package.UsesSwig()) && (cfg.BuildBuildmode == "c-archive" || cfg.BuildBuildmode == "c-shared") { + for _, t1 := range a.triggers { + if t1.Mode == "install header" { + return true + } + } + for _, t1 := range a.triggers { + for _, t2 := range t1.triggers { + if t2.Mode == "install header" { + return true + } + } + } + } + return false +} + +// allowedVersion reports whether the version v is an allowed version of go +// (one that we can compile). +// v is known to be of the form "1.23". +func allowedVersion(v string) bool { + // Special case: no requirement. + if v == "" { + return true + } + // Special case "1.0" means "go1", which is OK. + if v == "1.0" { + return true + } + // Otherwise look through release tags of form "go1.23" for one that matches. + for _, tag := range cfg.BuildContext.ReleaseTags { + if strings.HasPrefix(tag, "go") && tag[2:] == v { + return true + } + } + return false +} + +const ( + needBuild uint32 = 1 << iota + needCgoHdr + needVet + needCompiledGoFiles + needStale +) + +// build is the action for building a single package. +// Note that any new influence on this logic must be reported in b.buildActionID above as well. +func (b *Builder) build(ctx context.Context, a *Action) (err error) { + p := a.Package + + bit := func(x uint32, b bool) uint32 { + if b { + return x + } + return 0 + } + + cachedBuild := false + need := bit(needBuild, !b.IsCmdList && a.needBuild || b.NeedExport) | + bit(needCgoHdr, b.needCgoHdr(a)) | + bit(needVet, a.needVet) | + bit(needCompiledGoFiles, b.NeedCompiledGoFiles) + + if !p.BinaryOnly { + if b.useCache(a, b.buildActionID(a), p.Target) { + // We found the main output in the cache. + // If we don't need any other outputs, we can stop. + // Otherwise, we need to write files to a.Objdir (needVet, needCgoHdr). + // Remember that we might have them in cache + // and check again after we create a.Objdir. + cachedBuild = true + a.output = []byte{} // start saving output in case we miss any cache results + need &^= needBuild + if b.NeedExport { + p.Export = a.built + p.BuildID = a.buildID + } + if need&needCompiledGoFiles != 0 { + if err := b.loadCachedSrcFiles(a); err == nil { + need &^= needCompiledGoFiles + } + } + } + + // Source files might be cached, even if the full action is not + // (e.g., go list -compiled -find). + if !cachedBuild && need&needCompiledGoFiles != 0 { + if err := b.loadCachedSrcFiles(a); err == nil { + need &^= needCompiledGoFiles + } + } + + if need == 0 { + return nil + } + defer b.flushOutput(a) + } + + defer func() { + if err != nil && err != errPrintedOutput { + err = fmt.Errorf("go build %s: %v", a.Package.ImportPath, err) + } + if err != nil && b.IsCmdList && b.NeedError && p.Error == nil { + p.Error = &load.PackageError{Err: err} + } + }() + if cfg.BuildN { + // In -n mode, print a banner between packages. + // The banner is five lines so that when changes to + // different sections of the bootstrap script have to + // be merged, the banners give patch something + // to use to find its context. + b.Print("\n#\n# " + a.Package.ImportPath + "\n#\n\n") + } + + if cfg.BuildV { + b.Print(a.Package.ImportPath + "\n") + } + + if a.Package.BinaryOnly { + p.Stale = true + p.StaleReason = "binary-only packages are no longer supported" + if b.IsCmdList { + return nil + } + return errors.New("binary-only packages are no longer supported") + } + + if err := b.Mkdir(a.Objdir); err != nil { + return err + } + objdir := a.Objdir + + // Load cached cgo header, but only if we're skipping the main build (cachedBuild==true). + if cachedBuild && need&needCgoHdr != 0 { + if err := b.loadCachedCgoHdr(a); err == nil { + need &^= needCgoHdr + } + } + + // Load cached vet config, but only if that's all we have left + // (need == needVet, not testing just the one bit). + // If we are going to do a full build anyway, + // we're going to regenerate the files below anyway. + if need == needVet { + if err := b.loadCachedVet(a); err == nil { + need &^= needVet + } + } + if need == 0 { + return nil + } + + if err := allowInstall(a); err != nil { + return err + } + + // make target directory + dir, _ := filepath.Split(a.Target) + if dir != "" { + if err := b.Mkdir(dir); err != nil { + return err + } + } + + gofiles := str.StringList(a.Package.GoFiles) + cgofiles := str.StringList(a.Package.CgoFiles) + cfiles := str.StringList(a.Package.CFiles) + sfiles := str.StringList(a.Package.SFiles) + cxxfiles := str.StringList(a.Package.CXXFiles) + var objects, cgoObjects, pcCFLAGS, pcLDFLAGS []string + + if a.Package.UsesCgo() || a.Package.UsesSwig() { + if pcCFLAGS, pcLDFLAGS, err = b.getPkgConfigFlags(a.Package); err != nil { + return + } + } + + // Compute overlays for .c/.cc/.h/etc. and if there are any overlays + // put correct contents of all those files in the objdir, to ensure + // the correct headers are included. nonGoOverlay is the overlay that + // points from nongo files to the copied files in objdir. + nonGoFileLists := [][]string{a.Package.CFiles, a.Package.SFiles, a.Package.CXXFiles, a.Package.HFiles, a.Package.FFiles} +OverlayLoop: + for _, fs := range nonGoFileLists { + for _, f := range fs { + if _, ok := fsys.OverlayPath(mkAbs(p.Dir, f)); ok { + a.nonGoOverlay = make(map[string]string) + break OverlayLoop + } + } + } + if a.nonGoOverlay != nil { + for _, fs := range nonGoFileLists { + for i := range fs { + from := mkAbs(p.Dir, fs[i]) + opath, _ := fsys.OverlayPath(from) + dst := objdir + filepath.Base(fs[i]) + if err := b.copyFile(dst, opath, 0666, false); err != nil { + return err + } + a.nonGoOverlay[from] = dst + } + } + } + + // Run SWIG on each .swig and .swigcxx file. + // Each run will generate two files, a .go file and a .c or .cxx file. + // The .go file will use import "C" and is to be processed by cgo. + if a.Package.UsesSwig() { + outGo, outC, outCXX, err := b.swig(a, a.Package, objdir, pcCFLAGS) + if err != nil { + return err + } + cgofiles = append(cgofiles, outGo...) + cfiles = append(cfiles, outC...) + cxxfiles = append(cxxfiles, outCXX...) + } + + // If we're doing coverage, preprocess the .go files and put them in the work directory + if a.Package.Internal.CoverMode != "" { + for i, file := range str.StringList(gofiles, cgofiles) { + var sourceFile string + var coverFile string + var key string + if strings.HasSuffix(file, ".cgo1.go") { + // cgo files have absolute paths + base := filepath.Base(file) + sourceFile = file + coverFile = objdir + base + key = strings.TrimSuffix(base, ".cgo1.go") + ".go" + } else { + sourceFile = filepath.Join(a.Package.Dir, file) + coverFile = objdir + file + key = file + } + coverFile = strings.TrimSuffix(coverFile, ".go") + ".cover.go" + cover := a.Package.Internal.CoverVars[key] + if cover == nil || base.IsTestFile(file) { + // Not covering this file. + continue + } + if err := b.cover(a, coverFile, sourceFile, cover.Var); err != nil { + return err + } + if i < len(gofiles) { + gofiles[i] = coverFile + } else { + cgofiles[i-len(gofiles)] = coverFile + } + } + } + + // Run cgo. + if a.Package.UsesCgo() || a.Package.UsesSwig() { + // In a package using cgo, cgo compiles the C, C++ and assembly files with gcc. + // There is one exception: runtime/cgo's job is to bridge the + // cgo and non-cgo worlds, so it necessarily has files in both. + // In that case gcc only gets the gcc_* files. + var gccfiles []string + gccfiles = append(gccfiles, cfiles...) + cfiles = nil + if a.Package.Standard && a.Package.ImportPath == "runtime/cgo" { + filter := func(files, nongcc, gcc []string) ([]string, []string) { + for _, f := range files { + if strings.HasPrefix(f, "gcc_") { + gcc = append(gcc, f) + } else { + nongcc = append(nongcc, f) + } + } + return nongcc, gcc + } + sfiles, gccfiles = filter(sfiles, sfiles[:0], gccfiles) + } else { + for _, sfile := range sfiles { + data, err := os.ReadFile(filepath.Join(a.Package.Dir, sfile)) + if err == nil { + if bytes.HasPrefix(data, []byte("TEXT")) || bytes.Contains(data, []byte("\nTEXT")) || + bytes.HasPrefix(data, []byte("DATA")) || bytes.Contains(data, []byte("\nDATA")) || + bytes.HasPrefix(data, []byte("GLOBL")) || bytes.Contains(data, []byte("\nGLOBL")) { + return fmt.Errorf("package using cgo has Go assembly file %s", sfile) + } + } + } + gccfiles = append(gccfiles, sfiles...) + sfiles = nil + } + + outGo, outObj, err := b.cgo(a, base.Tool("cgo"), objdir, pcCFLAGS, pcLDFLAGS, mkAbsFiles(a.Package.Dir, cgofiles), gccfiles, cxxfiles, a.Package.MFiles, a.Package.FFiles) + if err != nil { + return err + } + if cfg.BuildToolchainName == "gccgo" { + cgoObjects = append(cgoObjects, a.Objdir+"_cgo_flags") + } + cgoObjects = append(cgoObjects, outObj...) + gofiles = append(gofiles, outGo...) + + switch cfg.BuildBuildmode { + case "c-archive", "c-shared": + b.cacheCgoHdr(a) + } + } + + var srcfiles []string // .go and non-.go + srcfiles = append(srcfiles, gofiles...) + srcfiles = append(srcfiles, sfiles...) + srcfiles = append(srcfiles, cfiles...) + srcfiles = append(srcfiles, cxxfiles...) + b.cacheSrcFiles(a, srcfiles) + + // Running cgo generated the cgo header. + need &^= needCgoHdr + + // Sanity check only, since Package.load already checked as well. + if len(gofiles) == 0 { + return &load.NoGoError{Package: a.Package} + } + + // Prepare Go vet config if needed. + if need&needVet != 0 { + buildVetConfig(a, srcfiles) + need &^= needVet + } + if need&needCompiledGoFiles != 0 { + if err := b.loadCachedSrcFiles(a); err != nil { + return fmt.Errorf("loading compiled Go files from cache: %w", err) + } + need &^= needCompiledGoFiles + } + if need == 0 { + // Nothing left to do. + return nil + } + + // Collect symbol ABI requirements from assembly. + symabis, err := BuildToolchain.symabis(b, a, sfiles) + if err != nil { + return err + } + + // Prepare Go import config. + // We start it off with a comment so it can't be empty, so icfg.Bytes() below is never nil. + // It should never be empty anyway, but there have been bugs in the past that resulted + // in empty configs, which then unfortunately turn into "no config passed to compiler", + // and the compiler falls back to looking in pkg itself, which mostly works, + // except when it doesn't. + var icfg bytes.Buffer + fmt.Fprintf(&icfg, "# import config\n") + for i, raw := range a.Package.Internal.RawImports { + final := a.Package.Imports[i] + if final != raw { + fmt.Fprintf(&icfg, "importmap %s=%s\n", raw, final) + } + } + for _, a1 := range a.Deps { + p1 := a1.Package + if p1 == nil || p1.ImportPath == "" || a1.built == "" { + continue + } + fmt.Fprintf(&icfg, "packagefile %s=%s\n", p1.ImportPath, a1.built) + } + + // Prepare Go embed config if needed. + // Unlike the import config, it's okay for the embed config to be empty. + var embedcfg []byte + if len(p.Internal.Embed) > 0 { + var embed struct { + Patterns map[string][]string + Files map[string]string + } + embed.Patterns = p.Internal.Embed + embed.Files = make(map[string]string) + for _, file := range p.EmbedFiles { + embed.Files[file] = filepath.Join(p.Dir, file) + } + js, err := json.MarshalIndent(&embed, "", "\t") + if err != nil { + return fmt.Errorf("marshal embedcfg: %v", err) + } + embedcfg = js + } + + if p.Internal.BuildInfo != "" && cfg.ModulesEnabled { + if err := b.writeFile(objdir+"_gomod_.go", modload.ModInfoProg(p.Internal.BuildInfo, cfg.BuildToolchainName == "gccgo")); err != nil { + return err + } + gofiles = append(gofiles, objdir+"_gomod_.go") + } + + // Compile Go. + objpkg := objdir + "_pkg_.a" + ofile, out, err := BuildToolchain.gc(b, a, objpkg, icfg.Bytes(), embedcfg, symabis, len(sfiles) > 0, gofiles) + if len(out) > 0 { + output := b.processOutput(out) + if p.Module != nil && !allowedVersion(p.Module.GoVersion) { + output += "note: module requires Go " + p.Module.GoVersion + "\n" + } + b.showOutput(a, a.Package.Dir, a.Package.Desc(), output) + if err != nil { + return errPrintedOutput + } + } + if err != nil { + if p.Module != nil && !allowedVersion(p.Module.GoVersion) { + b.showOutput(a, a.Package.Dir, a.Package.Desc(), "note: module requires Go "+p.Module.GoVersion+"\n") + } + return err + } + if ofile != objpkg { + objects = append(objects, ofile) + } + + // Copy .h files named for goos or goarch or goos_goarch + // to names using GOOS and GOARCH. + // For example, defs_linux_amd64.h becomes defs_GOOS_GOARCH.h. + _goos_goarch := "_" + cfg.Goos + "_" + cfg.Goarch + _goos := "_" + cfg.Goos + _goarch := "_" + cfg.Goarch + for _, file := range a.Package.HFiles { + name, ext := fileExtSplit(file) + switch { + case strings.HasSuffix(name, _goos_goarch): + targ := file[:len(name)-len(_goos_goarch)] + "_GOOS_GOARCH." + ext + if err := b.copyFile(objdir+targ, filepath.Join(a.Package.Dir, file), 0666, true); err != nil { + return err + } + case strings.HasSuffix(name, _goarch): + targ := file[:len(name)-len(_goarch)] + "_GOARCH." + ext + if err := b.copyFile(objdir+targ, filepath.Join(a.Package.Dir, file), 0666, true); err != nil { + return err + } + case strings.HasSuffix(name, _goos): + targ := file[:len(name)-len(_goos)] + "_GOOS." + ext + if err := b.copyFile(objdir+targ, filepath.Join(a.Package.Dir, file), 0666, true); err != nil { + return err + } + } + } + + for _, file := range cfiles { + out := file[:len(file)-len(".c")] + ".o" + if err := BuildToolchain.cc(b, a, objdir+out, file); err != nil { + return err + } + objects = append(objects, out) + } + + // Assemble .s files. + if len(sfiles) > 0 { + ofiles, err := BuildToolchain.asm(b, a, sfiles) + if err != nil { + return err + } + objects = append(objects, ofiles...) + } + + // For gccgo on ELF systems, we write the build ID as an assembler file. + // This lets us set the SHF_EXCLUDE flag. + // This is read by readGccgoArchive in cmd/internal/buildid/buildid.go. + if a.buildID != "" && cfg.BuildToolchainName == "gccgo" { + switch cfg.Goos { + case "aix", "android", "dragonfly", "freebsd", "illumos", "linux", "netbsd", "openbsd", "solaris": + asmfile, err := b.gccgoBuildIDFile(a) + if err != nil { + return err + } + ofiles, err := BuildToolchain.asm(b, a, []string{asmfile}) + if err != nil { + return err + } + objects = append(objects, ofiles...) + } + } + + // NOTE(rsc): On Windows, it is critically important that the + // gcc-compiled objects (cgoObjects) be listed after the ordinary + // objects in the archive. I do not know why this is. + // https://golang.org/issue/2601 + objects = append(objects, cgoObjects...) + + // Add system object files. + for _, syso := range a.Package.SysoFiles { + objects = append(objects, filepath.Join(a.Package.Dir, syso)) + } + + // Pack into archive in objdir directory. + // If the Go compiler wrote an archive, we only need to add the + // object files for non-Go sources to the archive. + // If the Go compiler wrote an archive and the package is entirely + // Go sources, there is no pack to execute at all. + if len(objects) > 0 { + if err := BuildToolchain.pack(b, a, objpkg, objects); err != nil { + return err + } + } + + if err := b.updateBuildID(a, objpkg, true); err != nil { + return err + } + + a.built = objpkg + return nil +} + +func (b *Builder) cacheObjdirFile(a *Action, c *cache.Cache, name string) error { + f, err := os.Open(a.Objdir + name) + if err != nil { + return err + } + defer f.Close() + _, _, err = c.Put(cache.Subkey(a.actionID, name), f) + return err +} + +func (b *Builder) findCachedObjdirFile(a *Action, c *cache.Cache, name string) (string, error) { + file, _, err := c.GetFile(cache.Subkey(a.actionID, name)) + if err != nil { + return "", fmt.Errorf("loading cached file %s: %w", name, err) + } + return file, nil +} + +func (b *Builder) loadCachedObjdirFile(a *Action, c *cache.Cache, name string) error { + cached, err := b.findCachedObjdirFile(a, c, name) + if err != nil { + return err + } + return b.copyFile(a.Objdir+name, cached, 0666, true) +} + +func (b *Builder) cacheCgoHdr(a *Action) { + c := cache.Default() + b.cacheObjdirFile(a, c, "_cgo_install.h") +} + +func (b *Builder) loadCachedCgoHdr(a *Action) error { + c := cache.Default() + return b.loadCachedObjdirFile(a, c, "_cgo_install.h") +} + +func (b *Builder) cacheSrcFiles(a *Action, srcfiles []string) { + c := cache.Default() + var buf bytes.Buffer + for _, file := range srcfiles { + if !strings.HasPrefix(file, a.Objdir) { + // not generated + buf.WriteString("./") + buf.WriteString(file) + buf.WriteString("\n") + continue + } + name := file[len(a.Objdir):] + buf.WriteString(name) + buf.WriteString("\n") + if err := b.cacheObjdirFile(a, c, name); err != nil { + return + } + } + c.PutBytes(cache.Subkey(a.actionID, "srcfiles"), buf.Bytes()) +} + +func (b *Builder) loadCachedVet(a *Action) error { + c := cache.Default() + list, _, err := c.GetBytes(cache.Subkey(a.actionID, "srcfiles")) + if err != nil { + return fmt.Errorf("reading srcfiles list: %w", err) + } + var srcfiles []string + for _, name := range strings.Split(string(list), "\n") { + if name == "" { // end of list + continue + } + if strings.HasPrefix(name, "./") { + srcfiles = append(srcfiles, name[2:]) + continue + } + if err := b.loadCachedObjdirFile(a, c, name); err != nil { + return err + } + srcfiles = append(srcfiles, a.Objdir+name) + } + buildVetConfig(a, srcfiles) + return nil +} + +func (b *Builder) loadCachedSrcFiles(a *Action) error { + c := cache.Default() + list, _, err := c.GetBytes(cache.Subkey(a.actionID, "srcfiles")) + if err != nil { + return fmt.Errorf("reading srcfiles list: %w", err) + } + var files []string + for _, name := range strings.Split(string(list), "\n") { + if name == "" { // end of list + continue + } + if strings.HasPrefix(name, "./") { + files = append(files, name[len("./"):]) + continue + } + file, err := b.findCachedObjdirFile(a, c, name) + if err != nil { + return fmt.Errorf("finding %s: %w", name, err) + } + files = append(files, file) + } + a.Package.CompiledGoFiles = files + return nil +} + +// vetConfig is the configuration passed to vet describing a single package. +type vetConfig struct { + ID string // package ID (example: "fmt [fmt.test]") + Compiler string // compiler name (gc, gccgo) + Dir string // directory containing package + ImportPath string // canonical import path ("package path") + GoFiles []string // absolute paths to package source files + NonGoFiles []string // absolute paths to package non-Go files + IgnoredFiles []string // absolute paths to ignored source files + + ImportMap map[string]string // map import path in source code to package path + PackageFile map[string]string // map package path to .a file with export data + Standard map[string]bool // map package path to whether it's in the standard library + PackageVetx map[string]string // map package path to vetx data from earlier vet run + VetxOnly bool // only compute vetx data; don't report detected problems + VetxOutput string // write vetx data to this output file + + SucceedOnTypecheckFailure bool // awful hack; see #18395 and below +} + +func buildVetConfig(a *Action, srcfiles []string) { + // Classify files based on .go extension. + // srcfiles does not include raw cgo files. + var gofiles, nongofiles []string + for _, name := range srcfiles { + if strings.HasSuffix(name, ".go") { + gofiles = append(gofiles, name) + } else { + nongofiles = append(nongofiles, name) + } + } + + ignored := str.StringList(a.Package.IgnoredGoFiles, a.Package.IgnoredOtherFiles) + + // Pass list of absolute paths to vet, + // so that vet's error messages will use absolute paths, + // so that we can reformat them relative to the directory + // in which the go command is invoked. + vcfg := &vetConfig{ + ID: a.Package.ImportPath, + Compiler: cfg.BuildToolchainName, + Dir: a.Package.Dir, + GoFiles: mkAbsFiles(a.Package.Dir, gofiles), + NonGoFiles: mkAbsFiles(a.Package.Dir, nongofiles), + IgnoredFiles: mkAbsFiles(a.Package.Dir, ignored), + ImportPath: a.Package.ImportPath, + ImportMap: make(map[string]string), + PackageFile: make(map[string]string), + Standard: make(map[string]bool), + } + a.vetCfg = vcfg + for i, raw := range a.Package.Internal.RawImports { + final := a.Package.Imports[i] + vcfg.ImportMap[raw] = final + } + + // Compute the list of mapped imports in the vet config + // so that we can add any missing mappings below. + vcfgMapped := make(map[string]bool) + for _, p := range vcfg.ImportMap { + vcfgMapped[p] = true + } + + for _, a1 := range a.Deps { + p1 := a1.Package + if p1 == nil || p1.ImportPath == "" { + continue + } + // Add import mapping if needed + // (for imports like "runtime/cgo" that appear only in generated code). + if !vcfgMapped[p1.ImportPath] { + vcfg.ImportMap[p1.ImportPath] = p1.ImportPath + } + if a1.built != "" { + vcfg.PackageFile[p1.ImportPath] = a1.built + } + if p1.Standard { + vcfg.Standard[p1.ImportPath] = true + } + } +} + +// VetTool is the path to an alternate vet tool binary. +// The caller is expected to set it (if needed) before executing any vet actions. +var VetTool string + +// VetFlags are the default flags to pass to vet. +// The caller is expected to set them before executing any vet actions. +var VetFlags []string + +// VetExplicit records whether the vet flags were set explicitly on the command line. +var VetExplicit bool + +func (b *Builder) vet(ctx context.Context, a *Action) error { + // a.Deps[0] is the build of the package being vetted. + // a.Deps[1] is the build of the "fmt" package. + + a.Failed = false // vet of dependency may have failed but we can still succeed + + if a.Deps[0].Failed { + // The build of the package has failed. Skip vet check. + // Vet could return export data for non-typecheck errors, + // but we ignore it because the package cannot be compiled. + return nil + } + + vcfg := a.Deps[0].vetCfg + if vcfg == nil { + // Vet config should only be missing if the build failed. + return fmt.Errorf("vet config not found") + } + + vcfg.VetxOnly = a.VetxOnly + vcfg.VetxOutput = a.Objdir + "vet.out" + vcfg.PackageVetx = make(map[string]string) + + h := cache.NewHash("vet " + a.Package.ImportPath) + fmt.Fprintf(h, "vet %q\n", b.toolID("vet")) + + vetFlags := VetFlags + + // In GOROOT, we enable all the vet tests during 'go test', + // not just the high-confidence subset. This gets us extra + // checking for the standard library (at some compliance cost) + // and helps us gain experience about how well the checks + // work, to help decide which should be turned on by default. + // The command-line still wins. + // + // Note that this flag change applies even when running vet as + // a dependency of vetting a package outside std. + // (Otherwise we'd have to introduce a whole separate + // space of "vet fmt as a dependency of a std top-level vet" + // versus "vet fmt as a dependency of a non-std top-level vet".) + // This is OK as long as the packages that are farther down the + // dependency tree turn on *more* analysis, as here. + // (The unsafeptr check does not write any facts for use by + // later vet runs, nor does unreachable.) + if a.Package.Goroot && !VetExplicit && VetTool == "" { + // Turn off -unsafeptr checks. + // There's too much unsafe.Pointer code + // that vet doesn't like in low-level packages + // like runtime, sync, and reflect. + // Note that $GOROOT/src/buildall.bash + // does the same for the misc-compile trybots + // and should be updated if these flags are + // changed here. + vetFlags = []string{"-unsafeptr=false"} + + // Also turn off -unreachable checks during go test. + // During testing it is very common to make changes + // like hard-coded forced returns or panics that make + // code unreachable. It's unreasonable to insist on files + // not having any unreachable code during "go test". + // (buildall.bash still runs with -unreachable enabled + // for the overall whole-tree scan.) + if cfg.CmdName == "test" { + vetFlags = append(vetFlags, "-unreachable=false") + } + } + + // Note: We could decide that vet should compute export data for + // all analyses, in which case we don't need to include the flags here. + // But that would mean that if an analysis causes problems like + // unexpected crashes there would be no way to turn it off. + // It seems better to let the flags disable export analysis too. + fmt.Fprintf(h, "vetflags %q\n", vetFlags) + + fmt.Fprintf(h, "pkg %q\n", a.Deps[0].actionID) + for _, a1 := range a.Deps { + if a1.Mode == "vet" && a1.built != "" { + fmt.Fprintf(h, "vetout %q %s\n", a1.Package.ImportPath, b.fileHash(a1.built)) + vcfg.PackageVetx[a1.Package.ImportPath] = a1.built + } + } + key := cache.ActionID(h.Sum()) + + if vcfg.VetxOnly && !cfg.BuildA { + c := cache.Default() + if file, _, err := c.GetFile(key); err == nil { + a.built = file + return nil + } + } + + js, err := json.MarshalIndent(vcfg, "", "\t") + if err != nil { + return fmt.Errorf("internal error marshaling vet config: %v", err) + } + js = append(js, '\n') + if err := b.writeFile(a.Objdir+"vet.cfg", js); err != nil { + return err + } + + // TODO(rsc): Why do we pass $GCCGO to go vet? + env := b.cCompilerEnv() + if cfg.BuildToolchainName == "gccgo" { + env = append(env, "GCCGO="+BuildToolchain.compiler()) + } + + p := a.Package + tool := VetTool + if tool == "" { + tool = base.Tool("vet") + } + runErr := b.run(a, p.Dir, p.ImportPath, env, cfg.BuildToolexec, tool, vetFlags, a.Objdir+"vet.cfg") + + // If vet wrote export data, save it for input to future vets. + if f, err := os.Open(vcfg.VetxOutput); err == nil { + a.built = vcfg.VetxOutput + cache.Default().Put(key, f) + f.Close() + } + + return runErr +} + +// linkActionID computes the action ID for a link action. +func (b *Builder) linkActionID(a *Action) cache.ActionID { + p := a.Package + h := cache.NewHash("link " + p.ImportPath) + + // Toolchain-independent configuration. + fmt.Fprintf(h, "link\n") + fmt.Fprintf(h, "buildmode %s goos %s goarch %s\n", cfg.BuildBuildmode, cfg.Goos, cfg.Goarch) + fmt.Fprintf(h, "import %q\n", p.ImportPath) + fmt.Fprintf(h, "omitdebug %v standard %v local %v prefix %q\n", p.Internal.OmitDebug, p.Standard, p.Internal.Local, p.Internal.LocalPrefix) + if cfg.BuildTrimpath { + fmt.Fprintln(h, "trimpath") + } + + // Toolchain-dependent configuration, shared with b.linkSharedActionID. + b.printLinkerConfig(h, p) + + // Input files. + for _, a1 := range a.Deps { + p1 := a1.Package + if p1 != nil { + if a1.built != "" || a1.buildID != "" { + buildID := a1.buildID + if buildID == "" { + buildID = b.buildID(a1.built) + } + fmt.Fprintf(h, "packagefile %s=%s\n", p1.ImportPath, contentID(buildID)) + } + // Because we put package main's full action ID into the binary's build ID, + // we must also put the full action ID into the binary's action ID hash. + if p1.Name == "main" { + fmt.Fprintf(h, "packagemain %s\n", a1.buildID) + } + if p1.Shlib != "" { + fmt.Fprintf(h, "packageshlib %s=%s\n", p1.ImportPath, contentID(b.buildID(p1.Shlib))) + } + } + } + + return h.Sum() +} + +// printLinkerConfig prints the linker config into the hash h, +// as part of the computation of a linker-related action ID. +func (b *Builder) printLinkerConfig(h io.Writer, p *load.Package) { + switch cfg.BuildToolchainName { + default: + base.Fatalf("linkActionID: unknown toolchain %q", cfg.BuildToolchainName) + + case "gc": + fmt.Fprintf(h, "link %s %q %s\n", b.toolID("link"), forcedLdflags, ldBuildmode) + if p != nil { + fmt.Fprintf(h, "linkflags %q\n", p.Internal.Ldflags) + } + + // GOARM, GOMIPS, etc. + key, val := cfg.GetArchEnv() + fmt.Fprintf(h, "%s=%s\n", key, val) + + // The linker writes source file paths that say GOROOT_FINAL, but + // only if -trimpath is not specified (see ld() in gc.go). + gorootFinal := cfg.GOROOT_FINAL + if cfg.BuildTrimpath { + gorootFinal = trimPathGoRootFinal + } + fmt.Fprintf(h, "GOROOT=%s\n", gorootFinal) + + // GO_EXTLINK_ENABLED controls whether the external linker is used. + fmt.Fprintf(h, "GO_EXTLINK_ENABLED=%s\n", cfg.Getenv("GO_EXTLINK_ENABLED")) + + // TODO(rsc): Do cgo settings and flags need to be included? + // Or external linker settings and flags? + + case "gccgo": + id, err := b.gccgoToolID(BuildToolchain.linker(), "go") + if err != nil { + base.Fatalf("%v", err) + } + fmt.Fprintf(h, "link %s %s\n", id, ldBuildmode) + // TODO(iant): Should probably include cgo flags here. + } +} + +// link is the action for linking a single command. +// Note that any new influence on this logic must be reported in b.linkActionID above as well. +func (b *Builder) link(ctx context.Context, a *Action) (err error) { + if b.useCache(a, b.linkActionID(a), a.Package.Target) || b.IsCmdList { + return nil + } + defer b.flushOutput(a) + + if err := b.Mkdir(a.Objdir); err != nil { + return err + } + + importcfg := a.Objdir + "importcfg.link" + if err := b.writeLinkImportcfg(a, importcfg); err != nil { + return err + } + + if err := allowInstall(a); err != nil { + return err + } + + // make target directory + dir, _ := filepath.Split(a.Target) + if dir != "" { + if err := b.Mkdir(dir); err != nil { + return err + } + } + + if err := BuildToolchain.ld(b, a, a.Target, importcfg, a.Deps[0].built); err != nil { + return err + } + + // Update the binary with the final build ID. + // But if OmitDebug is set, don't rewrite the binary, because we set OmitDebug + // on binaries that we are going to run and then delete. + // There's no point in doing work on such a binary. + // Worse, opening the binary for write here makes it + // essentially impossible to safely fork+exec due to a fundamental + // incompatibility between ETXTBSY and threads on modern Unix systems. + // See golang.org/issue/22220. + // We still call updateBuildID to update a.buildID, which is important + // for test result caching, but passing rewrite=false (final arg) + // means we don't actually rewrite the binary, nor store the + // result into the cache. That's probably a net win: + // less cache space wasted on large binaries we are not likely to + // need again. (On the other hand it does make repeated go test slower.) + // It also makes repeated go run slower, which is a win in itself: + // we don't want people to treat go run like a scripting environment. + if err := b.updateBuildID(a, a.Target, !a.Package.Internal.OmitDebug); err != nil { + return err + } + + a.built = a.Target + return nil +} + +func (b *Builder) writeLinkImportcfg(a *Action, file string) error { + // Prepare Go import cfg. + var icfg bytes.Buffer + for _, a1 := range a.Deps { + p1 := a1.Package + if p1 == nil { + continue + } + fmt.Fprintf(&icfg, "packagefile %s=%s\n", p1.ImportPath, a1.built) + if p1.Shlib != "" { + fmt.Fprintf(&icfg, "packageshlib %s=%s\n", p1.ImportPath, p1.Shlib) + } + } + return b.writeFile(file, icfg.Bytes()) +} + +// PkgconfigCmd returns a pkg-config binary name +// defaultPkgConfig is defined in zdefaultcc.go, written by cmd/dist. +func (b *Builder) PkgconfigCmd() string { + return envList("PKG_CONFIG", cfg.DefaultPkgConfig)[0] +} + +// splitPkgConfigOutput parses the pkg-config output into a slice of +// flags. This implements the algorithm from pkgconf/libpkgconf/argvsplit.c. +func splitPkgConfigOutput(out []byte) ([]string, error) { + if len(out) == 0 { + return nil, nil + } + var flags []string + flag := make([]byte, 0, len(out)) + escaped := false + quote := byte(0) + + for _, c := range out { + if escaped { + if quote != 0 { + switch c { + case '$', '`', '"', '\\': + default: + flag = append(flag, '\\') + } + flag = append(flag, c) + } else { + flag = append(flag, c) + } + escaped = false + } else if quote != 0 { + if c == quote { + quote = 0 + } else { + switch c { + case '\\': + escaped = true + default: + flag = append(flag, c) + } + } + } else if strings.IndexByte(" \t\n\v\f\r", c) < 0 { + switch c { + case '\\': + escaped = true + case '\'', '"': + quote = c + default: + flag = append(flag, c) + } + } else if len(flag) != 0 { + flags = append(flags, string(flag)) + flag = flag[:0] + } + } + if escaped { + return nil, errors.New("broken character escaping in pkgconf output ") + } + if quote != 0 { + return nil, errors.New("unterminated quoted string in pkgconf output ") + } else if len(flag) != 0 { + flags = append(flags, string(flag)) + } + + return flags, nil +} + +// Calls pkg-config if needed and returns the cflags/ldflags needed to build the package. +func (b *Builder) getPkgConfigFlags(p *load.Package) (cflags, ldflags []string, err error) { + if pcargs := p.CgoPkgConfig; len(pcargs) > 0 { + // pkg-config permits arguments to appear anywhere in + // the command line. Move them all to the front, before --. + var pcflags []string + var pkgs []string + for _, pcarg := range pcargs { + if pcarg == "--" { + // We're going to add our own "--" argument. + } else if strings.HasPrefix(pcarg, "--") { + pcflags = append(pcflags, pcarg) + } else { + pkgs = append(pkgs, pcarg) + } + } + for _, pkg := range pkgs { + if !load.SafeArg(pkg) { + return nil, nil, fmt.Errorf("invalid pkg-config package name: %s", pkg) + } + } + var out []byte + out, err = b.runOut(nil, p.Dir, nil, b.PkgconfigCmd(), "--cflags", pcflags, "--", pkgs) + if err != nil { + b.showOutput(nil, p.Dir, b.PkgconfigCmd()+" --cflags "+strings.Join(pcflags, " ")+" -- "+strings.Join(pkgs, " "), string(out)) + b.Print(err.Error() + "\n") + return nil, nil, errPrintedOutput + } + if len(out) > 0 { + cflags, err = splitPkgConfigOutput(out) + if err != nil { + return nil, nil, err + } + if err := checkCompilerFlags("CFLAGS", "pkg-config --cflags", cflags); err != nil { + return nil, nil, err + } + } + out, err = b.runOut(nil, p.Dir, nil, b.PkgconfigCmd(), "--libs", pcflags, "--", pkgs) + if err != nil { + b.showOutput(nil, p.Dir, b.PkgconfigCmd()+" --libs "+strings.Join(pcflags, " ")+" -- "+strings.Join(pkgs, " "), string(out)) + b.Print(err.Error() + "\n") + return nil, nil, errPrintedOutput + } + if len(out) > 0 { + ldflags = strings.Fields(string(out)) + if err := checkLinkerFlags("LDFLAGS", "pkg-config --libs", ldflags); err != nil { + return nil, nil, err + } + } + } + + return +} + +func (b *Builder) installShlibname(ctx context.Context, a *Action) error { + if err := allowInstall(a); err != nil { + return err + } + + // TODO: BuildN + a1 := a.Deps[0] + err := os.WriteFile(a.Target, []byte(filepath.Base(a1.Target)+"\n"), 0666) + if err != nil { + return err + } + if cfg.BuildX { + b.Showcmd("", "echo '%s' > %s # internal", filepath.Base(a1.Target), a.Target) + } + return nil +} + +func (b *Builder) linkSharedActionID(a *Action) cache.ActionID { + h := cache.NewHash("linkShared") + + // Toolchain-independent configuration. + fmt.Fprintf(h, "linkShared\n") + fmt.Fprintf(h, "goos %s goarch %s\n", cfg.Goos, cfg.Goarch) + + // Toolchain-dependent configuration, shared with b.linkActionID. + b.printLinkerConfig(h, nil) + + // Input files. + for _, a1 := range a.Deps { + p1 := a1.Package + if a1.built == "" { + continue + } + if p1 != nil { + fmt.Fprintf(h, "packagefile %s=%s\n", p1.ImportPath, contentID(b.buildID(a1.built))) + if p1.Shlib != "" { + fmt.Fprintf(h, "packageshlib %s=%s\n", p1.ImportPath, contentID(b.buildID(p1.Shlib))) + } + } + } + // Files named on command line are special. + for _, a1 := range a.Deps[0].Deps { + p1 := a1.Package + fmt.Fprintf(h, "top %s=%s\n", p1.ImportPath, contentID(b.buildID(a1.built))) + } + + return h.Sum() +} + +func (b *Builder) linkShared(ctx context.Context, a *Action) (err error) { + if b.useCache(a, b.linkSharedActionID(a), a.Target) || b.IsCmdList { + return nil + } + defer b.flushOutput(a) + + if err := allowInstall(a); err != nil { + return err + } + + if err := b.Mkdir(a.Objdir); err != nil { + return err + } + + importcfg := a.Objdir + "importcfg.link" + if err := b.writeLinkImportcfg(a, importcfg); err != nil { + return err + } + + // TODO(rsc): There is a missing updateBuildID here, + // but we have to decide where to store the build ID in these files. + a.built = a.Target + return BuildToolchain.ldShared(b, a, a.Deps[0].Deps, a.Target, importcfg, a.Deps) +} + +// BuildInstallFunc is the action for installing a single package or executable. +func BuildInstallFunc(b *Builder, ctx context.Context, a *Action) (err error) { + defer func() { + if err != nil && err != errPrintedOutput { + // a.Package == nil is possible for the go install -buildmode=shared + // action that installs libmangledname.so, which corresponds to + // a list of packages, not just one. + sep, path := "", "" + if a.Package != nil { + sep, path = " ", a.Package.ImportPath + } + err = fmt.Errorf("go %s%s%s: %v", cfg.CmdName, sep, path, err) + } + }() + + a1 := a.Deps[0] + a.buildID = a1.buildID + if a.json != nil { + a.json.BuildID = a.buildID + } + + // If we are using the eventual install target as an up-to-date + // cached copy of the thing we built, then there's no need to + // copy it into itself (and that would probably fail anyway). + // In this case a1.built == a.Target because a1.built == p.Target, + // so the built target is not in the a1.Objdir tree that b.cleanup(a1) removes. + if a1.built == a.Target { + a.built = a.Target + if !a.buggyInstall { + b.cleanup(a1) + } + // Whether we're smart enough to avoid a complete rebuild + // depends on exactly what the staleness and rebuild algorithms + // are, as well as potentially the state of the Go build cache. + // We don't really want users to be able to infer (or worse start depending on) + // those details from whether the modification time changes during + // "go install", so do a best-effort update of the file times to make it + // look like we rewrote a.Target even if we did not. Updating the mtime + // may also help other mtime-based systems that depend on our + // previous mtime updates that happened more often. + // This is still not perfect - we ignore the error result, and if the file was + // unwritable for some reason then pretending to have written it is also + // confusing - but it's probably better than not doing the mtime update. + // + // But don't do that for the special case where building an executable + // with -linkshared implicitly installs all its dependent libraries. + // We want to hide that awful detail as much as possible, so don't + // advertise it by touching the mtimes (usually the libraries are up + // to date). + if !a.buggyInstall && !b.IsCmdList { + if cfg.BuildN { + b.Showcmd("", "touch %s", a.Target) + } else if err := allowInstall(a); err == nil { + now := time.Now() + os.Chtimes(a.Target, now, now) + } + } + return nil + } + + // If we're building for go list -export, + // never install anything; just keep the cache reference. + if b.IsCmdList { + a.built = a1.built + return nil + } + if err := allowInstall(a); err != nil { + return err + } + + if err := b.Mkdir(a.Objdir); err != nil { + return err + } + + perm := fs.FileMode(0666) + if a1.Mode == "link" { + switch cfg.BuildBuildmode { + case "c-archive", "c-shared", "plugin": + default: + perm = 0777 + } + } + + // make target directory + dir, _ := filepath.Split(a.Target) + if dir != "" { + if err := b.Mkdir(dir); err != nil { + return err + } + } + + if !a.buggyInstall { + defer b.cleanup(a1) + } + + return b.moveOrCopyFile(a.Target, a1.built, perm, false) +} + +// allowInstall returns a non-nil error if this invocation of the go command is +// allowed to install a.Target. +// +// (The build of cmd/go running under its own test is forbidden from installing +// to its original GOROOT.) +var allowInstall = func(*Action) error { return nil } + +// cleanup removes a's object dir to keep the amount of +// on-disk garbage down in a large build. On an operating system +// with aggressive buffering, cleaning incrementally like +// this keeps the intermediate objects from hitting the disk. +func (b *Builder) cleanup(a *Action) { + if !cfg.BuildWork { + if cfg.BuildX { + // Don't say we are removing the directory if + // we never created it. + if _, err := os.Stat(a.Objdir); err == nil || cfg.BuildN { + b.Showcmd("", "rm -r %s", a.Objdir) + } + } + os.RemoveAll(a.Objdir) + } +} + +// moveOrCopyFile is like 'mv src dst' or 'cp src dst'. +func (b *Builder) moveOrCopyFile(dst, src string, perm fs.FileMode, force bool) error { + if cfg.BuildN { + b.Showcmd("", "mv %s %s", src, dst) + return nil + } + + // If we can update the mode and rename to the dst, do it. + // Otherwise fall back to standard copy. + + // If the source is in the build cache, we need to copy it. + if strings.HasPrefix(src, cache.DefaultDir()) { + return b.copyFile(dst, src, perm, force) + } + + // On Windows, always copy the file, so that we respect the NTFS + // permissions of the parent folder. https://golang.org/issue/22343. + // What matters here is not cfg.Goos (the system we are building + // for) but runtime.GOOS (the system we are building on). + if runtime.GOOS == "windows" { + return b.copyFile(dst, src, perm, force) + } + + // If the destination directory has the group sticky bit set, + // we have to copy the file to retain the correct permissions. + // https://golang.org/issue/18878 + if fi, err := os.Stat(filepath.Dir(dst)); err == nil { + if fi.IsDir() && (fi.Mode()&fs.ModeSetgid) != 0 { + return b.copyFile(dst, src, perm, force) + } + } + + // The perm argument is meant to be adjusted according to umask, + // but we don't know what the umask is. + // Create a dummy file to find out. + // This avoids build tags and works even on systems like Plan 9 + // where the file mask computation incorporates other information. + mode := perm + f, err := os.OpenFile(filepath.Clean(dst)+"-go-tmp-umask", os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm) + if err == nil { + fi, err := f.Stat() + if err == nil { + mode = fi.Mode() & 0777 + } + name := f.Name() + f.Close() + os.Remove(name) + } + + if err := os.Chmod(src, mode); err == nil { + if err := os.Rename(src, dst); err == nil { + if cfg.BuildX { + b.Showcmd("", "mv %s %s", src, dst) + } + return nil + } + } + + return b.copyFile(dst, src, perm, force) +} + +// copyFile is like 'cp src dst'. +func (b *Builder) copyFile(dst, src string, perm fs.FileMode, force bool) error { + if cfg.BuildN || cfg.BuildX { + b.Showcmd("", "cp %s %s", src, dst) + if cfg.BuildN { + return nil + } + } + + sf, err := os.Open(src) + if err != nil { + return err + } + defer sf.Close() + + // Be careful about removing/overwriting dst. + // Do not remove/overwrite if dst exists and is a directory + // or a non-empty non-object file. + if fi, err := os.Stat(dst); err == nil { + if fi.IsDir() { + return fmt.Errorf("build output %q already exists and is a directory", dst) + } + if !force && fi.Mode().IsRegular() && fi.Size() != 0 && !isObject(dst) { + return fmt.Errorf("build output %q already exists and is not an object file", dst) + } + } + + // On Windows, remove lingering ~ file from last attempt. + if base.ToolIsWindows { + if _, err := os.Stat(dst + "~"); err == nil { + os.Remove(dst + "~") + } + } + + mayberemovefile(dst) + df, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil && base.ToolIsWindows { + // Windows does not allow deletion of a binary file + // while it is executing. Try to move it out of the way. + // If the move fails, which is likely, we'll try again the + // next time we do an install of this binary. + if err := os.Rename(dst, dst+"~"); err == nil { + os.Remove(dst + "~") + } + df, err = os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + } + if err != nil { + return fmt.Errorf("copying %s: %w", src, err) // err should already refer to dst + } + + _, err = io.Copy(df, sf) + df.Close() + if err != nil { + mayberemovefile(dst) + return fmt.Errorf("copying %s to %s: %v", src, dst, err) + } + return nil +} + +// writeFile writes the text to file. +func (b *Builder) writeFile(file string, text []byte) error { + if cfg.BuildN || cfg.BuildX { + b.Showcmd("", "cat >%s << 'EOF' # internal\n%sEOF", file, text) + } + if cfg.BuildN { + return nil + } + return os.WriteFile(file, text, 0666) +} + +// Install the cgo export header file, if there is one. +func (b *Builder) installHeader(ctx context.Context, a *Action) error { + src := a.Objdir + "_cgo_install.h" + if _, err := os.Stat(src); os.IsNotExist(err) { + // If the file does not exist, there are no exported + // functions, and we do not install anything. + // TODO(rsc): Once we know that caching is rebuilding + // at the right times (not missing rebuilds), here we should + // probably delete the installed header, if any. + if cfg.BuildX { + b.Showcmd("", "# %s not created", src) + } + return nil + } + + if err := allowInstall(a); err != nil { + return err + } + + dir, _ := filepath.Split(a.Target) + if dir != "" { + if err := b.Mkdir(dir); err != nil { + return err + } + } + + return b.moveOrCopyFile(a.Target, src, 0666, true) +} + +// cover runs, in effect, +// go tool cover -mode=b.coverMode -var="varName" -o dst.go src.go +func (b *Builder) cover(a *Action, dst, src string, varName string) error { + return b.run(a, a.Objdir, "cover "+a.Package.ImportPath, nil, + cfg.BuildToolexec, + base.Tool("cover"), + "-mode", a.Package.Internal.CoverMode, + "-var", varName, + "-o", dst, + src) +} + +var objectMagic = [][]byte{ + {'!', '<', 'a', 'r', 'c', 'h', '>', '\n'}, // Package archive + {'<', 'b', 'i', 'g', 'a', 'f', '>', '\n'}, // Package AIX big archive + {'\x7F', 'E', 'L', 'F'}, // ELF + {0xFE, 0xED, 0xFA, 0xCE}, // Mach-O big-endian 32-bit + {0xFE, 0xED, 0xFA, 0xCF}, // Mach-O big-endian 64-bit + {0xCE, 0xFA, 0xED, 0xFE}, // Mach-O little-endian 32-bit + {0xCF, 0xFA, 0xED, 0xFE}, // Mach-O little-endian 64-bit + {0x4d, 0x5a, 0x90, 0x00, 0x03, 0x00}, // PE (Windows) as generated by 6l/8l and gcc + {0x00, 0x00, 0x01, 0xEB}, // Plan 9 i386 + {0x00, 0x00, 0x8a, 0x97}, // Plan 9 amd64 + {0x00, 0x00, 0x06, 0x47}, // Plan 9 arm + {0x00, 0x61, 0x73, 0x6D}, // WASM + {0x01, 0xDF}, // XCOFF 32bit + {0x01, 0xF7}, // XCOFF 64bit +} + +func isObject(s string) bool { + f, err := os.Open(s) + if err != nil { + return false + } + defer f.Close() + buf := make([]byte, 64) + io.ReadFull(f, buf) + for _, magic := range objectMagic { + if bytes.HasPrefix(buf, magic) { + return true + } + } + return false +} + +// mayberemovefile removes a file only if it is a regular file +// When running as a user with sufficient privileges, we may delete +// even device files, for example, which is not intended. +func mayberemovefile(s string) { + if fi, err := os.Lstat(s); err == nil && !fi.Mode().IsRegular() { + return + } + os.Remove(s) +} + +// fmtcmd formats a command in the manner of fmt.Sprintf but also: +// +// If dir is non-empty and the script is not in dir right now, +// fmtcmd inserts "cd dir\n" before the command. +// +// fmtcmd replaces the value of b.WorkDir with $WORK. +// fmtcmd replaces the value of goroot with $GOROOT. +// fmtcmd replaces the value of b.gobin with $GOBIN. +// +// fmtcmd replaces the name of the current directory with dot (.) +// but only when it is at the beginning of a space-separated token. +// +func (b *Builder) fmtcmd(dir string, format string, args ...interface{}) string { + cmd := fmt.Sprintf(format, args...) + if dir != "" && dir != "/" { + dot := " ." + if dir[len(dir)-1] == filepath.Separator { + dot += string(filepath.Separator) + } + cmd = strings.ReplaceAll(" "+cmd, " "+dir, dot)[1:] + if b.scriptDir != dir { + b.scriptDir = dir + cmd = "cd " + dir + "\n" + cmd + } + } + if b.WorkDir != "" { + cmd = strings.ReplaceAll(cmd, b.WorkDir, "$WORK") + escaped := strconv.Quote(b.WorkDir) + escaped = escaped[1 : len(escaped)-1] // strip quote characters + if escaped != b.WorkDir { + cmd = strings.ReplaceAll(cmd, escaped, "$WORK") + } + } + return cmd +} + +// showcmd prints the given command to standard output +// for the implementation of -n or -x. +func (b *Builder) Showcmd(dir string, format string, args ...interface{}) { + b.output.Lock() + defer b.output.Unlock() + b.Print(b.fmtcmd(dir, format, args...) + "\n") +} + +// showOutput prints "# desc" followed by the given output. +// The output is expected to contain references to 'dir', usually +// the source directory for the package that has failed to build. +// showOutput rewrites mentions of dir with a relative path to dir +// when the relative path is shorter. This is usually more pleasant. +// For example, if fmt doesn't compile and we are in src/html, +// the output is +// +// $ go build +// # fmt +// ../fmt/print.go:1090: undefined: asdf +// $ +// +// instead of +// +// $ go build +// # fmt +// /usr/gopher/go/src/fmt/print.go:1090: undefined: asdf +// $ +// +// showOutput also replaces references to the work directory with $WORK. +// +// If a is not nil and a.output is not nil, showOutput appends to that slice instead of +// printing to b.Print. +// +func (b *Builder) showOutput(a *Action, dir, desc, out string) { + prefix := "# " + desc + suffix := "\n" + out + if reldir := base.ShortPath(dir); reldir != dir { + suffix = strings.ReplaceAll(suffix, " "+dir, " "+reldir) + suffix = strings.ReplaceAll(suffix, "\n"+dir, "\n"+reldir) + } + suffix = strings.ReplaceAll(suffix, " "+b.WorkDir, " $WORK") + + if a != nil && a.output != nil { + a.output = append(a.output, prefix...) + a.output = append(a.output, suffix...) + return + } + + b.output.Lock() + defer b.output.Unlock() + b.Print(prefix, suffix) +} + +// errPrintedOutput is a special error indicating that a command failed +// but that it generated output as well, and that output has already +// been printed, so there's no point showing 'exit status 1' or whatever +// the wait status was. The main executor, builder.do, knows not to +// print this error. +var errPrintedOutput = errors.New("already printed output - no need to show error") + +var cgoLine = lazyregexp.New(`\[[^\[\]]+\.(cgo1|cover)\.go:[0-9]+(:[0-9]+)?\]`) +var cgoTypeSigRe = lazyregexp.New(`\b_C2?(type|func|var|macro)_\B`) + +// run runs the command given by cmdline in the directory dir. +// If the command fails, run prints information about the failure +// and returns a non-nil error. +func (b *Builder) run(a *Action, dir string, desc string, env []string, cmdargs ...interface{}) error { + out, err := b.runOut(a, dir, env, cmdargs...) + if len(out) > 0 { + if desc == "" { + desc = b.fmtcmd(dir, "%s", strings.Join(str.StringList(cmdargs...), " ")) + } + b.showOutput(a, dir, desc, b.processOutput(out)) + if err != nil { + err = errPrintedOutput + } + } + return err +} + +// processOutput prepares the output of runOut to be output to the console. +func (b *Builder) processOutput(out []byte) string { + if out[len(out)-1] != '\n' { + out = append(out, '\n') + } + messages := string(out) + // Fix up output referring to cgo-generated code to be more readable. + // Replace x.go:19[/tmp/.../x.cgo1.go:18] with x.go:19. + // Replace *[100]_Ctype_foo with *[100]C.foo. + // If we're using -x, assume we're debugging and want the full dump, so disable the rewrite. + if !cfg.BuildX && cgoLine.MatchString(messages) { + messages = cgoLine.ReplaceAllString(messages, "") + messages = cgoTypeSigRe.ReplaceAllString(messages, "C.") + } + return messages +} + +// runOut runs the command given by cmdline in the directory dir. +// It returns the command output and any errors that occurred. +// It accumulates execution time in a. +func (b *Builder) runOut(a *Action, dir string, env []string, cmdargs ...interface{}) ([]byte, error) { + cmdline := str.StringList(cmdargs...) + + for _, arg := range cmdline { + // GNU binutils commands, including gcc and gccgo, interpret an argument + // @foo anywhere in the command line (even following --) as meaning + // "read and insert arguments from the file named foo." + // Don't say anything that might be misinterpreted that way. + if strings.HasPrefix(arg, "@") { + return nil, fmt.Errorf("invalid command-line argument %s in command: %s", arg, joinUnambiguously(cmdline)) + } + } + + if cfg.BuildN || cfg.BuildX { + var envcmdline string + for _, e := range env { + if j := strings.IndexByte(e, '='); j != -1 { + if strings.ContainsRune(e[j+1:], '\'') { + envcmdline += fmt.Sprintf("%s=%q", e[:j], e[j+1:]) + } else { + envcmdline += fmt.Sprintf("%s='%s'", e[:j], e[j+1:]) + } + envcmdline += " " + } + } + envcmdline += joinUnambiguously(cmdline) + b.Showcmd(dir, "%s", envcmdline) + if cfg.BuildN { + return nil, nil + } + } + + var buf bytes.Buffer + cmd := exec.Command(cmdline[0], cmdline[1:]...) + if cmd.Path != "" { + cmd.Args[0] = cmd.Path + } + cmd.Stdout = &buf + cmd.Stderr = &buf + cleanup := passLongArgsInResponseFiles(cmd) + defer cleanup() + cmd.Dir = dir + cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir) + + // Add the TOOLEXEC_IMPORTPATH environment variable for -toolexec tools. + // It doesn't really matter if -toolexec isn't being used. + if a != nil && a.Package != nil { + cmd.Env = append(cmd.Env, "TOOLEXEC_IMPORTPATH="+a.Package.ImportPath) + } + + cmd.Env = append(cmd.Env, env...) + start := time.Now() + err := cmd.Run() + if a != nil && a.json != nil { + aj := a.json + aj.Cmd = append(aj.Cmd, joinUnambiguously(cmdline)) + aj.CmdReal += time.Since(start) + if ps := cmd.ProcessState; ps != nil { + aj.CmdUser += ps.UserTime() + aj.CmdSys += ps.SystemTime() + } + } + + // err can be something like 'exit status 1'. + // Add information about what program was running. + // Note that if buf.Bytes() is non-empty, the caller usually + // shows buf.Bytes() and does not print err at all, so the + // prefix here does not make most output any more verbose. + if err != nil { + err = errors.New(cmdline[0] + ": " + err.Error()) + } + return buf.Bytes(), err +} + +// joinUnambiguously prints the slice, quoting where necessary to make the +// output unambiguous. +// TODO: See issue 5279. The printing of commands needs a complete redo. +func joinUnambiguously(a []string) string { + var buf bytes.Buffer + for i, s := range a { + if i > 0 { + buf.WriteByte(' ') + } + q := strconv.Quote(s) + // A gccgo command line can contain -( and -). + // Make sure we quote them since they are special to the shell. + // The trimpath argument can also contain > (part of =>) and ;. Quote those too. + if s == "" || strings.ContainsAny(s, " ()>;") || len(q) > len(s)+2 { + buf.WriteString(q) + } else { + buf.WriteString(s) + } + } + return buf.String() +} + +// cCompilerEnv returns environment variables to set when running the +// C compiler. This is needed to disable escape codes in clang error +// messages that confuse tools like cgo. +func (b *Builder) cCompilerEnv() []string { + return []string{"TERM=dumb"} +} + +// mkdir makes the named directory. +func (b *Builder) Mkdir(dir string) error { + // Make Mkdir(a.Objdir) a no-op instead of an error when a.Objdir == "". + if dir == "" { + return nil + } + + b.exec.Lock() + defer b.exec.Unlock() + // We can be a little aggressive about being + // sure directories exist. Skip repeated calls. + if b.mkdirCache[dir] { + return nil + } + b.mkdirCache[dir] = true + + if cfg.BuildN || cfg.BuildX { + b.Showcmd("", "mkdir -p %s", dir) + if cfg.BuildN { + return nil + } + } + + if err := os.MkdirAll(dir, 0777); err != nil { + return err + } + return nil +} + +// symlink creates a symlink newname -> oldname. +func (b *Builder) Symlink(oldname, newname string) error { + // It's not an error to try to recreate an existing symlink. + if link, err := os.Readlink(newname); err == nil && link == oldname { + return nil + } + + if cfg.BuildN || cfg.BuildX { + b.Showcmd("", "ln -s %s %s", oldname, newname) + if cfg.BuildN { + return nil + } + } + return os.Symlink(oldname, newname) +} + +// mkAbs returns an absolute path corresponding to +// evaluating f in the directory dir. +// We always pass absolute paths of source files so that +// the error messages will include the full path to a file +// in need of attention. +func mkAbs(dir, f string) string { + // Leave absolute paths alone. + // Also, during -n mode we use the pseudo-directory $WORK + // instead of creating an actual work directory that won't be used. + // Leave paths beginning with $WORK alone too. + if filepath.IsAbs(f) || strings.HasPrefix(f, "$WORK") { + return f + } + return filepath.Join(dir, f) +} + +type toolchain interface { + // gc runs the compiler in a specific directory on a set of files + // and returns the name of the generated output file. + gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, out []byte, err error) + // cc runs the toolchain's C compiler in a directory on a C file + // to produce an output file. + cc(b *Builder, a *Action, ofile, cfile string) error + // asm runs the assembler in a specific directory on specific files + // and returns a list of named output files. + asm(b *Builder, a *Action, sfiles []string) ([]string, error) + // symabis scans the symbol ABIs from sfiles and returns the + // path to the output symbol ABIs file, or "" if none. + symabis(b *Builder, a *Action, sfiles []string) (string, error) + // pack runs the archive packer in a specific directory to create + // an archive from a set of object files. + // typically it is run in the object directory. + pack(b *Builder, a *Action, afile string, ofiles []string) error + // ld runs the linker to create an executable starting at mainpkg. + ld(b *Builder, root *Action, out, importcfg, mainpkg string) error + // ldShared runs the linker to create a shared library containing the pkgs built by toplevelactions + ldShared(b *Builder, root *Action, toplevelactions []*Action, out, importcfg string, allactions []*Action) error + + compiler() string + linker() string +} + +type noToolchain struct{} + +func noCompiler() error { + log.Fatalf("unknown compiler %q", cfg.BuildContext.Compiler) + return nil +} + +func (noToolchain) compiler() string { + noCompiler() + return "" +} + +func (noToolchain) linker() string { + noCompiler() + return "" +} + +func (noToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, out []byte, err error) { + return "", nil, noCompiler() +} + +func (noToolchain) asm(b *Builder, a *Action, sfiles []string) ([]string, error) { + return nil, noCompiler() +} + +func (noToolchain) symabis(b *Builder, a *Action, sfiles []string) (string, error) { + return "", noCompiler() +} + +func (noToolchain) pack(b *Builder, a *Action, afile string, ofiles []string) error { + return noCompiler() +} + +func (noToolchain) ld(b *Builder, root *Action, out, importcfg, mainpkg string) error { + return noCompiler() +} + +func (noToolchain) ldShared(b *Builder, root *Action, toplevelactions []*Action, out, importcfg string, allactions []*Action) error { + return noCompiler() +} + +func (noToolchain) cc(b *Builder, a *Action, ofile, cfile string) error { + return noCompiler() +} + +// gcc runs the gcc C compiler to create an object from a single C file. +func (b *Builder) gcc(a *Action, p *load.Package, workdir, out string, flags []string, cfile string) error { + return b.ccompile(a, p, out, flags, cfile, b.GccCmd(p.Dir, workdir)) +} + +// gxx runs the g++ C++ compiler to create an object from a single C++ file. +func (b *Builder) gxx(a *Action, p *load.Package, workdir, out string, flags []string, cxxfile string) error { + return b.ccompile(a, p, out, flags, cxxfile, b.GxxCmd(p.Dir, workdir)) +} + +// gfortran runs the gfortran Fortran compiler to create an object from a single Fortran file. +func (b *Builder) gfortran(a *Action, p *load.Package, workdir, out string, flags []string, ffile string) error { + return b.ccompile(a, p, out, flags, ffile, b.gfortranCmd(p.Dir, workdir)) +} + +// ccompile runs the given C or C++ compiler and creates an object from a single source file. +func (b *Builder) ccompile(a *Action, p *load.Package, outfile string, flags []string, file string, compiler []string) error { + file = mkAbs(p.Dir, file) + desc := p.ImportPath + outfile = mkAbs(p.Dir, outfile) + + // Elide source directory paths if -trimpath or GOROOT_FINAL is set. + // This is needed for source files (e.g., a .c file in a package directory). + // TODO(golang.org/issue/36072): cgo also generates files with #line + // directives pointing to the source directory. It should not generate those + // when -trimpath is enabled. + if b.gccSupportsFlag(compiler, "-fdebug-prefix-map=a=b") { + if cfg.BuildTrimpath { + // Keep in sync with Action.trimpath. + // The trimmed paths are a little different, but we need to trim in the + // same situations. + var from, toPath string + if m := p.Module; m != nil { + from = m.Dir + toPath = m.Path + "@" + m.Version + } else { + from = p.Dir + toPath = p.ImportPath + } + // -fdebug-prefix-map requires an absolute "to" path (or it joins the path + // with the working directory). Pick something that makes sense for the + // target platform. + var to string + if cfg.BuildContext.GOOS == "windows" { + to = filepath.Join(`\\_\_`, toPath) + } else { + to = filepath.Join("/_", toPath) + } + flags = append(flags[:len(flags):len(flags)], "-fdebug-prefix-map="+from+"="+to) + } else if p.Goroot && cfg.GOROOT_FINAL != cfg.GOROOT { + flags = append(flags[:len(flags):len(flags)], "-fdebug-prefix-map="+cfg.GOROOT+"="+cfg.GOROOT_FINAL) + } + } + + overlayPath := file + if p, ok := a.nonGoOverlay[overlayPath]; ok { + overlayPath = p + } + output, err := b.runOut(a, filepath.Dir(overlayPath), b.cCompilerEnv(), compiler, flags, "-o", outfile, "-c", filepath.Base(overlayPath)) + if len(output) > 0 { + // On FreeBSD 11, when we pass -g to clang 3.8 it + // invokes its internal assembler with -dwarf-version=2. + // When it sees .section .note.GNU-stack, it warns + // "DWARF2 only supports one section per compilation unit". + // This warning makes no sense, since the section is empty, + // but it confuses people. + // We work around the problem by detecting the warning + // and dropping -g and trying again. + if bytes.Contains(output, []byte("DWARF2 only supports one section per compilation unit")) { + newFlags := make([]string, 0, len(flags)) + for _, f := range flags { + if !strings.HasPrefix(f, "-g") { + newFlags = append(newFlags, f) + } + } + if len(newFlags) < len(flags) { + return b.ccompile(a, p, outfile, newFlags, file, compiler) + } + } + + b.showOutput(a, p.Dir, desc, b.processOutput(output)) + if err != nil { + err = errPrintedOutput + } else if os.Getenv("GO_BUILDER_NAME") != "" { + return errors.New("C compiler warning promoted to error on Go builders") + } + } + return err +} + +// gccld runs the gcc linker to create an executable from a set of object files. +func (b *Builder) gccld(a *Action, p *load.Package, objdir, outfile string, flags []string, objs []string) error { + var cmd []string + if len(p.CXXFiles) > 0 || len(p.SwigCXXFiles) > 0 { + cmd = b.GxxCmd(p.Dir, objdir) + } else { + cmd = b.GccCmd(p.Dir, objdir) + } + + cmdargs := []interface{}{cmd, "-o", outfile, objs, flags} + dir := p.Dir + out, err := b.runOut(a, base.Cwd, b.cCompilerEnv(), cmdargs...) + + if len(out) > 0 { + // Filter out useless linker warnings caused by bugs outside Go. + // See also cmd/link/internal/ld's hostlink method. + var save [][]byte + var skipLines int + for _, line := range bytes.SplitAfter(out, []byte("\n")) { + // golang.org/issue/26073 - Apple Xcode bug + if bytes.Contains(line, []byte("ld: warning: text-based stub file")) { + continue + } + + if skipLines > 0 { + skipLines-- + continue + } + + // Remove duplicate main symbol with runtime/cgo on AIX. + // With runtime/cgo, two main are available: + // One is generated by cgo tool with {return 0;}. + // The other one is the main calling runtime.rt0_go + // in runtime/cgo. + // The second can't be used by cgo programs because + // runtime.rt0_go is unknown to them. + // Therefore, we let ld remove this main version + // and used the cgo generated one. + if p.ImportPath == "runtime/cgo" && bytes.Contains(line, []byte("ld: 0711-224 WARNING: Duplicate symbol: .main")) { + skipLines = 1 + continue + } + + save = append(save, line) + } + out = bytes.Join(save, nil) + if len(out) > 0 { + b.showOutput(nil, dir, p.ImportPath, b.processOutput(out)) + if err != nil { + err = errPrintedOutput + } + } + } + return err +} + +// Grab these before main helpfully overwrites them. +var ( + origCC = cfg.Getenv("CC") + origCXX = cfg.Getenv("CXX") +) + +// gccCmd returns a gcc command line prefix +// defaultCC is defined in zdefaultcc.go, written by cmd/dist. +func (b *Builder) GccCmd(incdir, workdir string) []string { + return b.compilerCmd(b.ccExe(), incdir, workdir) +} + +// gxxCmd returns a g++ command line prefix +// defaultCXX is defined in zdefaultcc.go, written by cmd/dist. +func (b *Builder) GxxCmd(incdir, workdir string) []string { + return b.compilerCmd(b.cxxExe(), incdir, workdir) +} + +// gfortranCmd returns a gfortran command line prefix. +func (b *Builder) gfortranCmd(incdir, workdir string) []string { + return b.compilerCmd(b.fcExe(), incdir, workdir) +} + +// ccExe returns the CC compiler setting without all the extra flags we add implicitly. +func (b *Builder) ccExe() []string { + return b.compilerExe(origCC, cfg.DefaultCC(cfg.Goos, cfg.Goarch)) +} + +// cxxExe returns the CXX compiler setting without all the extra flags we add implicitly. +func (b *Builder) cxxExe() []string { + return b.compilerExe(origCXX, cfg.DefaultCXX(cfg.Goos, cfg.Goarch)) +} + +// fcExe returns the FC compiler setting without all the extra flags we add implicitly. +func (b *Builder) fcExe() []string { + return b.compilerExe(cfg.Getenv("FC"), "gfortran") +} + +// compilerExe returns the compiler to use given an +// environment variable setting (the value not the name) +// and a default. The resulting slice is usually just the name +// of the compiler but can have additional arguments if they +// were present in the environment value. +// For example if CC="gcc -DGOPHER" then the result is ["gcc", "-DGOPHER"]. +func (b *Builder) compilerExe(envValue string, def string) []string { + compiler := strings.Fields(envValue) + if len(compiler) == 0 { + compiler = strings.Fields(def) + } + return compiler +} + +// compilerCmd returns a command line prefix for the given environment +// variable and using the default command when the variable is empty. +func (b *Builder) compilerCmd(compiler []string, incdir, workdir string) []string { + // NOTE: env.go's mkEnv knows that the first three + // strings returned are "gcc", "-I", incdir (and cuts them off). + a := []string{compiler[0], "-I", incdir} + a = append(a, compiler[1:]...) + + // Definitely want -fPIC but on Windows gcc complains + // "-fPIC ignored for target (all code is position independent)" + if cfg.Goos != "windows" { + a = append(a, "-fPIC") + } + a = append(a, b.gccArchArgs()...) + // gcc-4.5 and beyond require explicit "-pthread" flag + // for multithreading with pthread library. + if cfg.BuildContext.CgoEnabled { + switch cfg.Goos { + case "windows": + a = append(a, "-mthreads") + default: + a = append(a, "-pthread") + } + } + + if cfg.Goos == "aix" { + // mcmodel=large must always be enabled to allow large TOC. + a = append(a, "-mcmodel=large") + } + + // disable ASCII art in clang errors, if possible + if b.gccSupportsFlag(compiler, "-fno-caret-diagnostics") { + a = append(a, "-fno-caret-diagnostics") + } + // clang is too smart about command-line arguments + if b.gccSupportsFlag(compiler, "-Qunused-arguments") { + a = append(a, "-Qunused-arguments") + } + + // disable word wrapping in error messages + a = append(a, "-fmessage-length=0") + + // Tell gcc not to include the work directory in object files. + if b.gccSupportsFlag(compiler, "-fdebug-prefix-map=a=b") { + if workdir == "" { + workdir = b.WorkDir + } + workdir = strings.TrimSuffix(workdir, string(filepath.Separator)) + a = append(a, "-fdebug-prefix-map="+workdir+"=/tmp/go-build") + } + + // Tell gcc not to include flags in object files, which defeats the + // point of -fdebug-prefix-map above. + if b.gccSupportsFlag(compiler, "-gno-record-gcc-switches") { + a = append(a, "-gno-record-gcc-switches") + } + + // On OS X, some of the compilers behave as if -fno-common + // is always set, and the Mach-O linker in 6l/8l assumes this. + // See https://golang.org/issue/3253. + if cfg.Goos == "darwin" || cfg.Goos == "ios" { + a = append(a, "-fno-common") + } + + return a +} + +// gccNoPie returns the flag to use to request non-PIE. On systems +// with PIE (position independent executables) enabled by default, +// -no-pie must be passed when doing a partial link with -Wl,-r. +// But -no-pie is not supported by all compilers, and clang spells it -nopie. +func (b *Builder) gccNoPie(linker []string) string { + if b.gccSupportsFlag(linker, "-no-pie") { + return "-no-pie" + } + if b.gccSupportsFlag(linker, "-nopie") { + return "-nopie" + } + return "" +} + +// gccSupportsFlag checks to see if the compiler supports a flag. +func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool { + key := [2]string{compiler[0], flag} + + b.exec.Lock() + defer b.exec.Unlock() + if b, ok := b.flagCache[key]; ok { + return b + } + if b.flagCache == nil { + b.flagCache = make(map[[2]string]bool) + } + + tmp := os.DevNull + if runtime.GOOS == "windows" { + f, err := os.CreateTemp(b.WorkDir, "") + if err != nil { + return false + } + f.Close() + tmp = f.Name() + defer os.Remove(tmp) + } + + // We used to write an empty C file, but that gets complicated with + // go build -n. We tried using a file that does not exist, but that + // fails on systems with GCC version 4.2.1; that is the last GPLv2 + // version of GCC, so some systems have frozen on it. + // Now we pass an empty file on stdin, which should work at least for + // GCC and clang. + cmdArgs := str.StringList(compiler, flag, "-c", "-x", "c", "-", "-o", tmp) + if cfg.BuildN || cfg.BuildX { + b.Showcmd(b.WorkDir, "%s || true", joinUnambiguously(cmdArgs)) + if cfg.BuildN { + return false + } + } + cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...) + cmd.Dir = b.WorkDir + cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir) + cmd.Env = append(cmd.Env, "LC_ALL=C") + out, _ := cmd.CombinedOutput() + // GCC says "unrecognized command line option". + // clang says "unknown argument". + // Older versions of GCC say "unrecognised debug output level". + // For -fsplit-stack GCC says "'-fsplit-stack' is not supported". + supported := !bytes.Contains(out, []byte("unrecognized")) && + !bytes.Contains(out, []byte("unknown")) && + !bytes.Contains(out, []byte("unrecognised")) && + !bytes.Contains(out, []byte("is not supported")) + b.flagCache[key] = supported + return supported +} + +// gccArchArgs returns arguments to pass to gcc based on the architecture. +func (b *Builder) gccArchArgs() []string { + switch cfg.Goarch { + case "386": + return []string{"-m32"} + case "amd64": + if cfg.Goos == "darwin" { + return []string{"-arch", "x86_64", "-m64"} + } + return []string{"-m64"} + case "arm64": + if cfg.Goos == "darwin" { + return []string{"-arch", "arm64"} + } + case "arm": + return []string{"-marm"} // not thumb + case "s390x": + return []string{"-m64", "-march=z196"} + case "mips64", "mips64le": + return []string{"-mabi=64"} + case "mips", "mipsle": + return []string{"-mabi=32", "-march=mips32"} + case "ppc64": + if cfg.Goos == "aix" { + return []string{"-maix64"} + } + } + return nil +} + +// envList returns the value of the given environment variable broken +// into fields, using the default value when the variable is empty. +func envList(key, def string) []string { + v := cfg.Getenv(key) + if v == "" { + v = def + } + return strings.Fields(v) +} + +// CFlags returns the flags to use when invoking the C, C++ or Fortran compilers, or cgo. +func (b *Builder) CFlags(p *load.Package) (cppflags, cflags, cxxflags, fflags, ldflags []string, err error) { + defaults := "-g -O2" + + if cppflags, err = buildFlags("CPPFLAGS", "", p.CgoCPPFLAGS, checkCompilerFlags); err != nil { + return + } + if cflags, err = buildFlags("CFLAGS", defaults, p.CgoCFLAGS, checkCompilerFlags); err != nil { + return + } + if cxxflags, err = buildFlags("CXXFLAGS", defaults, p.CgoCXXFLAGS, checkCompilerFlags); err != nil { + return + } + if fflags, err = buildFlags("FFLAGS", defaults, p.CgoFFLAGS, checkCompilerFlags); err != nil { + return + } + if ldflags, err = buildFlags("LDFLAGS", defaults, p.CgoLDFLAGS, checkLinkerFlags); err != nil { + return + } + + return +} + +func buildFlags(name, defaults string, fromPackage []string, check func(string, string, []string) error) ([]string, error) { + if err := check(name, "#cgo "+name, fromPackage); err != nil { + return nil, err + } + return str.StringList(envList("CGO_"+name, defaults), fromPackage), nil +} + +var cgoRe = lazyregexp.New(`[/\\:]`) + +func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgofiles, gccfiles, gxxfiles, mfiles, ffiles []string) (outGo, outObj []string, err error) { + p := a.Package + cgoCPPFLAGS, cgoCFLAGS, cgoCXXFLAGS, cgoFFLAGS, cgoLDFLAGS, err := b.CFlags(p) + if err != nil { + return nil, nil, err + } + + cgoCPPFLAGS = append(cgoCPPFLAGS, pcCFLAGS...) + cgoLDFLAGS = append(cgoLDFLAGS, pcLDFLAGS...) + // If we are compiling Objective-C code, then we need to link against libobjc + if len(mfiles) > 0 { + cgoLDFLAGS = append(cgoLDFLAGS, "-lobjc") + } + + // Likewise for Fortran, except there are many Fortran compilers. + // Support gfortran out of the box and let others pass the correct link options + // via CGO_LDFLAGS + if len(ffiles) > 0 { + fc := cfg.Getenv("FC") + if fc == "" { + fc = "gfortran" + } + if strings.Contains(fc, "gfortran") { + cgoLDFLAGS = append(cgoLDFLAGS, "-lgfortran") + } + } + + if cfg.BuildMSan { + cgoCFLAGS = append([]string{"-fsanitize=memory"}, cgoCFLAGS...) + cgoLDFLAGS = append([]string{"-fsanitize=memory"}, cgoLDFLAGS...) + } + + // Allows including _cgo_export.h, as well as the user's .h files, + // from .[ch] files in the package. + cgoCPPFLAGS = append(cgoCPPFLAGS, "-I", objdir) + + // cgo + // TODO: CGO_FLAGS? + gofiles := []string{objdir + "_cgo_gotypes.go"} + cfiles := []string{"_cgo_export.c"} + for _, fn := range cgofiles { + f := strings.TrimSuffix(filepath.Base(fn), ".go") + gofiles = append(gofiles, objdir+f+".cgo1.go") + cfiles = append(cfiles, f+".cgo2.c") + } + + // TODO: make cgo not depend on $GOARCH? + + cgoflags := []string{} + if p.Standard && p.ImportPath == "runtime/cgo" { + cgoflags = append(cgoflags, "-import_runtime_cgo=false") + } + if p.Standard && (p.ImportPath == "runtime/race" || p.ImportPath == "runtime/msan" || p.ImportPath == "runtime/cgo") { + cgoflags = append(cgoflags, "-import_syscall=false") + } + + // Update $CGO_LDFLAGS with p.CgoLDFLAGS. + // These flags are recorded in the generated _cgo_gotypes.go file + // using //go:cgo_ldflag directives, the compiler records them in the + // object file for the package, and then the Go linker passes them + // along to the host linker. At this point in the code, cgoLDFLAGS + // consists of the original $CGO_LDFLAGS (unchecked) and all the + // flags put together from source code (checked). + cgoenv := b.cCompilerEnv() + if len(cgoLDFLAGS) > 0 { + flags := make([]string, len(cgoLDFLAGS)) + for i, f := range cgoLDFLAGS { + flags[i] = strconv.Quote(f) + } + cgoenv = append(cgoenv, "CGO_LDFLAGS="+strings.Join(flags, " ")) + } + + if cfg.BuildToolchainName == "gccgo" { + if b.gccSupportsFlag([]string{BuildToolchain.compiler()}, "-fsplit-stack") { + cgoCFLAGS = append(cgoCFLAGS, "-fsplit-stack") + } + cgoflags = append(cgoflags, "-gccgo") + if pkgpath := gccgoPkgpath(p); pkgpath != "" { + cgoflags = append(cgoflags, "-gccgopkgpath="+pkgpath) + } + } + + switch cfg.BuildBuildmode { + case "c-archive", "c-shared": + // Tell cgo that if there are any exported functions + // it should generate a header file that C code can + // #include. + cgoflags = append(cgoflags, "-exportheader="+objdir+"_cgo_install.h") + } + + execdir := p.Dir + + // Rewrite overlaid paths in cgo files. + // cgo adds //line and #line pragmas in generated files with these paths. + var trimpath []string + for i := range cgofiles { + path := mkAbs(p.Dir, cgofiles[i]) + if opath, ok := fsys.OverlayPath(path); ok { + cgofiles[i] = opath + trimpath = append(trimpath, opath+"=>"+path) + } + } + if len(trimpath) > 0 { + cgoflags = append(cgoflags, "-trimpath", strings.Join(trimpath, ";")) + } + + if err := b.run(a, execdir, p.ImportPath, cgoenv, cfg.BuildToolexec, cgoExe, "-objdir", objdir, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil { + return nil, nil, err + } + outGo = append(outGo, gofiles...) + + // Use sequential object file names to keep them distinct + // and short enough to fit in the .a header file name slots. + // We no longer collect them all into _all.o, and we'd like + // tools to see both the .o suffix and unique names, so + // we need to make them short enough not to be truncated + // in the final archive. + oseq := 0 + nextOfile := func() string { + oseq++ + return objdir + fmt.Sprintf("_x%03d.o", oseq) + } + + // gcc + cflags := str.StringList(cgoCPPFLAGS, cgoCFLAGS) + for _, cfile := range cfiles { + ofile := nextOfile() + if err := b.gcc(a, p, a.Objdir, ofile, cflags, objdir+cfile); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + for _, file := range gccfiles { + ofile := nextOfile() + if err := b.gcc(a, p, a.Objdir, ofile, cflags, file); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + cxxflags := str.StringList(cgoCPPFLAGS, cgoCXXFLAGS) + for _, file := range gxxfiles { + ofile := nextOfile() + if err := b.gxx(a, p, a.Objdir, ofile, cxxflags, file); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + for _, file := range mfiles { + ofile := nextOfile() + if err := b.gcc(a, p, a.Objdir, ofile, cflags, file); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + fflags := str.StringList(cgoCPPFLAGS, cgoFFLAGS) + for _, file := range ffiles { + ofile := nextOfile() + if err := b.gfortran(a, p, a.Objdir, ofile, fflags, file); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + switch cfg.BuildToolchainName { + case "gc": + importGo := objdir + "_cgo_import.go" + if err := b.dynimport(a, p, objdir, importGo, cgoExe, cflags, cgoLDFLAGS, outObj); err != nil { + return nil, nil, err + } + outGo = append(outGo, importGo) + + case "gccgo": + defunC := objdir + "_cgo_defun.c" + defunObj := objdir + "_cgo_defun.o" + if err := BuildToolchain.cc(b, a, defunObj, defunC); err != nil { + return nil, nil, err + } + outObj = append(outObj, defunObj) + + default: + noCompiler() + } + + // Double check the //go:cgo_ldflag comments in the generated files. + // The compiler only permits such comments in files whose base name + // starts with "_cgo_". Make sure that the comments in those files + // are safe. This is a backstop against people somehow smuggling + // such a comment into a file generated by cgo. + if cfg.BuildToolchainName == "gc" && !cfg.BuildN { + var flags []string + for _, f := range outGo { + if !strings.HasPrefix(filepath.Base(f), "_cgo_") { + continue + } + + src, err := os.ReadFile(f) + if err != nil { + return nil, nil, err + } + + const cgoLdflag = "//go:cgo_ldflag" + idx := bytes.Index(src, []byte(cgoLdflag)) + for idx >= 0 { + // We are looking at //go:cgo_ldflag. + // Find start of line. + start := bytes.LastIndex(src[:idx], []byte("\n")) + if start == -1 { + start = 0 + } + + // Find end of line. + end := bytes.Index(src[idx:], []byte("\n")) + if end == -1 { + end = len(src) + } else { + end += idx + } + + // Check for first line comment in line. + // We don't worry about /* */ comments, + // which normally won't appear in files + // generated by cgo. + commentStart := bytes.Index(src[start:], []byte("//")) + commentStart += start + // If that line comment is //go:cgo_ldflag, + // it's a match. + if bytes.HasPrefix(src[commentStart:], []byte(cgoLdflag)) { + // Pull out the flag, and unquote it. + // This is what the compiler does. + flag := string(src[idx+len(cgoLdflag) : end]) + flag = strings.TrimSpace(flag) + flag = strings.Trim(flag, `"`) + flags = append(flags, flag) + } + src = src[end:] + idx = bytes.Index(src, []byte(cgoLdflag)) + } + } + + // We expect to find the contents of cgoLDFLAGS in flags. + if len(cgoLDFLAGS) > 0 { + outer: + for i := range flags { + for j, f := range cgoLDFLAGS { + if f != flags[i+j] { + continue outer + } + } + flags = append(flags[:i], flags[i+len(cgoLDFLAGS):]...) + break + } + } + + if err := checkLinkerFlags("LDFLAGS", "go:cgo_ldflag", flags); err != nil { + return nil, nil, err + } + } + + return outGo, outObj, nil +} + +// dynimport creates a Go source file named importGo containing +// //go:cgo_import_dynamic directives for each symbol or library +// dynamically imported by the object files outObj. +func (b *Builder) dynimport(a *Action, p *load.Package, objdir, importGo, cgoExe string, cflags, cgoLDFLAGS, outObj []string) error { + cfile := objdir + "_cgo_main.c" + ofile := objdir + "_cgo_main.o" + if err := b.gcc(a, p, objdir, ofile, cflags, cfile); err != nil { + return err + } + + linkobj := str.StringList(ofile, outObj, mkAbsFiles(p.Dir, p.SysoFiles)) + dynobj := objdir + "_cgo_.o" + + // we need to use -pie for Linux/ARM to get accurate imported sym + ldflags := cgoLDFLAGS + if (cfg.Goarch == "arm" && cfg.Goos == "linux") || cfg.Goos == "android" { + // -static -pie doesn't make sense, and causes link errors. + // Issue 26197. + n := make([]string, 0, len(ldflags)) + for _, flag := range ldflags { + if flag != "-static" { + n = append(n, flag) + } + } + ldflags = append(n, "-pie") + } + if err := b.gccld(a, p, objdir, dynobj, ldflags, linkobj); err != nil { + return err + } + + // cgo -dynimport + var cgoflags []string + if p.Standard && p.ImportPath == "runtime/cgo" { + cgoflags = []string{"-dynlinker"} // record path to dynamic linker + } + return b.run(a, base.Cwd, p.ImportPath, b.cCompilerEnv(), cfg.BuildToolexec, cgoExe, "-dynpackage", p.Name, "-dynimport", dynobj, "-dynout", importGo, cgoflags) +} + +// Run SWIG on all SWIG input files. +// TODO: Don't build a shared library, once SWIG emits the necessary +// pragmas for external linking. +func (b *Builder) swig(a *Action, p *load.Package, objdir string, pcCFLAGS []string) (outGo, outC, outCXX []string, err error) { + if err := b.swigVersionCheck(); err != nil { + return nil, nil, nil, err + } + + intgosize, err := b.swigIntSize(objdir) + if err != nil { + return nil, nil, nil, err + } + + for _, f := range p.SwigFiles { + goFile, cFile, err := b.swigOne(a, p, f, objdir, pcCFLAGS, false, intgosize) + if err != nil { + return nil, nil, nil, err + } + if goFile != "" { + outGo = append(outGo, goFile) + } + if cFile != "" { + outC = append(outC, cFile) + } + } + for _, f := range p.SwigCXXFiles { + goFile, cxxFile, err := b.swigOne(a, p, f, objdir, pcCFLAGS, true, intgosize) + if err != nil { + return nil, nil, nil, err + } + if goFile != "" { + outGo = append(outGo, goFile) + } + if cxxFile != "" { + outCXX = append(outCXX, cxxFile) + } + } + return outGo, outC, outCXX, nil +} + +// Make sure SWIG is new enough. +var ( + swigCheckOnce sync.Once + swigCheck error +) + +func (b *Builder) swigDoVersionCheck() error { + out, err := b.runOut(nil, "", nil, "swig", "-version") + if err != nil { + return err + } + re := regexp.MustCompile(`[vV]ersion +([\d]+)([.][\d]+)?([.][\d]+)?`) + matches := re.FindSubmatch(out) + if matches == nil { + // Can't find version number; hope for the best. + return nil + } + + major, err := strconv.Atoi(string(matches[1])) + if err != nil { + // Can't find version number; hope for the best. + return nil + } + const errmsg = "must have SWIG version >= 3.0.6" + if major < 3 { + return errors.New(errmsg) + } + if major > 3 { + // 4.0 or later + return nil + } + + // We have SWIG version 3.x. + if len(matches[2]) > 0 { + minor, err := strconv.Atoi(string(matches[2][1:])) + if err != nil { + return nil + } + if minor > 0 { + // 3.1 or later + return nil + } + } + + // We have SWIG version 3.0.x. + if len(matches[3]) > 0 { + patch, err := strconv.Atoi(string(matches[3][1:])) + if err != nil { + return nil + } + if patch < 6 { + // Before 3.0.6. + return errors.New(errmsg) + } + } + + return nil +} + +func (b *Builder) swigVersionCheck() error { + swigCheckOnce.Do(func() { + swigCheck = b.swigDoVersionCheck() + }) + return swigCheck +} + +// Find the value to pass for the -intgosize option to swig. +var ( + swigIntSizeOnce sync.Once + swigIntSize string + swigIntSizeError error +) + +// This code fails to build if sizeof(int) <= 32 +const swigIntSizeCode = ` +package main +const i int = 1 << 32 +` + +// Determine the size of int on the target system for the -intgosize option +// of swig >= 2.0.9. Run only once. +func (b *Builder) swigDoIntSize(objdir string) (intsize string, err error) { + if cfg.BuildN { + return "$INTBITS", nil + } + src := filepath.Join(b.WorkDir, "swig_intsize.go") + if err = os.WriteFile(src, []byte(swigIntSizeCode), 0666); err != nil { + return + } + srcs := []string{src} + + p := load.GoFilesPackage(context.TODO(), srcs) + + if _, _, e := BuildToolchain.gc(b, &Action{Mode: "swigDoIntSize", Package: p, Objdir: objdir}, "", nil, nil, "", false, srcs); e != nil { + return "32", nil + } + return "64", nil +} + +// Determine the size of int on the target system for the -intgosize option +// of swig >= 2.0.9. +func (b *Builder) swigIntSize(objdir string) (intsize string, err error) { + swigIntSizeOnce.Do(func() { + swigIntSize, swigIntSizeError = b.swigDoIntSize(objdir) + }) + return swigIntSize, swigIntSizeError +} + +// Run SWIG on one SWIG input file. +func (b *Builder) swigOne(a *Action, p *load.Package, file, objdir string, pcCFLAGS []string, cxx bool, intgosize string) (outGo, outC string, err error) { + cgoCPPFLAGS, cgoCFLAGS, cgoCXXFLAGS, _, _, err := b.CFlags(p) + if err != nil { + return "", "", err + } + + var cflags []string + if cxx { + cflags = str.StringList(cgoCPPFLAGS, pcCFLAGS, cgoCXXFLAGS) + } else { + cflags = str.StringList(cgoCPPFLAGS, pcCFLAGS, cgoCFLAGS) + } + + n := 5 // length of ".swig" + if cxx { + n = 8 // length of ".swigcxx" + } + base := file[:len(file)-n] + goFile := base + ".go" + gccBase := base + "_wrap." + gccExt := "c" + if cxx { + gccExt = "cxx" + } + + gccgo := cfg.BuildToolchainName == "gccgo" + + // swig + args := []string{ + "-go", + "-cgo", + "-intgosize", intgosize, + "-module", base, + "-o", objdir + gccBase + gccExt, + "-outdir", objdir, + } + + for _, f := range cflags { + if len(f) > 3 && f[:2] == "-I" { + args = append(args, f) + } + } + + if gccgo { + args = append(args, "-gccgo") + if pkgpath := gccgoPkgpath(p); pkgpath != "" { + args = append(args, "-go-pkgpath", pkgpath) + } + } + if cxx { + args = append(args, "-c++") + } + + out, err := b.runOut(a, p.Dir, nil, "swig", args, file) + if err != nil { + if len(out) > 0 { + if bytes.Contains(out, []byte("-intgosize")) || bytes.Contains(out, []byte("-cgo")) { + return "", "", errors.New("must have SWIG version >= 3.0.6") + } + b.showOutput(a, p.Dir, p.Desc(), b.processOutput(out)) // swig error + return "", "", errPrintedOutput + } + return "", "", err + } + if len(out) > 0 { + b.showOutput(a, p.Dir, p.Desc(), b.processOutput(out)) // swig warning + } + + // If the input was x.swig, the output is x.go in the objdir. + // But there might be an x.go in the original dir too, and if it + // uses cgo as well, cgo will be processing both and will + // translate both into x.cgo1.go in the objdir, overwriting one. + // Rename x.go to _x_swig.go to avoid this problem. + // We ignore files in the original dir that begin with underscore + // so _x_swig.go cannot conflict with an original file we were + // going to compile. + goFile = objdir + goFile + newGoFile := objdir + "_" + base + "_swig.go" + if err := os.Rename(goFile, newGoFile); err != nil { + return "", "", err + } + return newGoFile, objdir + gccBase + gccExt, nil +} + +// disableBuildID adjusts a linker command line to avoid creating a +// build ID when creating an object file rather than an executable or +// shared library. Some systems, such as Ubuntu, always add +// --build-id to every link, but we don't want a build ID when we are +// producing an object file. On some of those system a plain -r (not +// -Wl,-r) will turn off --build-id, but clang 3.0 doesn't support a +// plain -r. I don't know how to turn off --build-id when using clang +// other than passing a trailing --build-id=none. So that is what we +// do, but only on systems likely to support it, which is to say, +// systems that normally use gold or the GNU linker. +func (b *Builder) disableBuildID(ldflags []string) []string { + switch cfg.Goos { + case "android", "dragonfly", "linux", "netbsd": + ldflags = append(ldflags, "-Wl,--build-id=none") + } + return ldflags +} + +// mkAbsFiles converts files into a list of absolute files, +// assuming they were originally relative to dir, +// and returns that new list. +func mkAbsFiles(dir string, files []string) []string { + abs := make([]string, len(files)) + for i, f := range files { + if !filepath.IsAbs(f) { + f = filepath.Join(dir, f) + } + abs[i] = f + } + return abs +} + +// passLongArgsInResponseFiles modifies cmd such that, for +// certain programs, long arguments are passed in "response files", a +// file on disk with the arguments, with one arg per line. An actual +// argument starting with '@' means that the rest of the argument is +// a filename of arguments to expand. +// +// See issues 18468 (Windows) and 37768 (Darwin). +func passLongArgsInResponseFiles(cmd *exec.Cmd) (cleanup func()) { + cleanup = func() {} // no cleanup by default + + var argLen int + for _, arg := range cmd.Args { + argLen += len(arg) + } + + // If we're not approaching 32KB of args, just pass args normally. + // (use 30KB instead to be conservative; not sure how accounting is done) + if !useResponseFile(cmd.Path, argLen) { + return + } + + tf, err := os.CreateTemp("", "args") + if err != nil { + log.Fatalf("error writing long arguments to response file: %v", err) + } + cleanup = func() { os.Remove(tf.Name()) } + var buf bytes.Buffer + for _, arg := range cmd.Args[1:] { + fmt.Fprintf(&buf, "%s\n", encodeArg(arg)) + } + if _, err := tf.Write(buf.Bytes()); err != nil { + tf.Close() + cleanup() + log.Fatalf("error writing long arguments to response file: %v", err) + } + if err := tf.Close(); err != nil { + cleanup() + log.Fatalf("error writing long arguments to response file: %v", err) + } + cmd.Args = []string{cmd.Args[0], "@" + tf.Name()} + return cleanup +} + +// Windows has a limit of 32 KB arguments. To be conservative and not worry +// about whether that includes spaces or not, just use 30 KB. Darwin's limit is +// less clear. The OS claims 256KB, but we've seen failures with arglen as +// small as 50KB. +const ArgLengthForResponseFile = (30 << 10) + +func useResponseFile(path string, argLen int) bool { + // Unless the program uses objabi.Flagparse, which understands + // response files, don't use response files. + // TODO: do we need more commands? asm? cgo? For now, no. + prog := strings.TrimSuffix(filepath.Base(path), ".exe") + switch prog { + case "compile", "link": + default: + return false + } + + if argLen > ArgLengthForResponseFile { + return true + } + + // On the Go build system, use response files about 10% of the + // time, just to exercise this codepath. + isBuilder := os.Getenv("GO_BUILDER_NAME") != "" + if isBuilder && rand.Intn(10) == 0 { + return true + } + + return false +} + +// encodeArg encodes an argument for response file writing. +func encodeArg(arg string) string { + // If there aren't any characters we need to reencode, fastpath out. + if !strings.ContainsAny(arg, "\\\n") { + return arg + } + var b strings.Builder + for _, r := range arg { + switch r { + case '\\': + b.WriteByte('\\') + b.WriteByte('\\') + case '\n': + b.WriteByte('\\') + b.WriteByte('n') + default: + b.WriteRune(r) + } + } + return b.String() +} diff --git a/src/cmd/go/internal/work/exec_test.go b/src/cmd/go/internal/work/exec_test.go new file mode 100644 index 0000000..4eb762c --- /dev/null +++ b/src/cmd/go/internal/work/exec_test.go @@ -0,0 +1,86 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "bytes" + "cmd/internal/objabi" + "fmt" + "math/rand" + "testing" + "time" + "unicode/utf8" +) + +func TestEncodeArgs(t *testing.T) { + t.Parallel() + tests := []struct { + arg, want string + }{ + {"", ""}, + {"hello", "hello"}, + {"hello\n", "hello\\n"}, + {"hello\\", "hello\\\\"}, + {"hello\nthere", "hello\\nthere"}, + {"\\\n", "\\\\\\n"}, + } + for _, test := range tests { + if got := encodeArg(test.arg); got != test.want { + t.Errorf("encodeArg(%q) = %q, want %q", test.arg, got, test.want) + } + } +} + +func TestEncodeDecode(t *testing.T) { + t.Parallel() + tests := []string{ + "", + "hello", + "hello\\there", + "hello\nthere", + "hello 中国", + "hello \n中\\国", + } + for _, arg := range tests { + if got := objabi.DecodeArg(encodeArg(arg)); got != arg { + t.Errorf("objabi.DecodeArg(encodeArg(%q)) = %q", arg, got) + } + } +} + +func TestEncodeDecodeFuzz(t *testing.T) { + if testing.Short() { + t.Skip("fuzz test is slow") + } + t.Parallel() + + nRunes := ArgLengthForResponseFile + 100 + rBuffer := make([]rune, nRunes) + buf := bytes.NewBuffer([]byte(string(rBuffer))) + + seed := time.Now().UnixNano() + t.Logf("rand seed: %v", seed) + rng := rand.New(rand.NewSource(seed)) + + for i := 0; i < 50; i++ { + // Generate a random string of runes. + buf.Reset() + for buf.Len() < ArgLengthForResponseFile+1 { + var r rune + for { + r = rune(rng.Intn(utf8.MaxRune + 1)) + if utf8.ValidRune(r) { + break + } + } + fmt.Fprintf(buf, "%c", r) + } + arg := buf.String() + + if got := objabi.DecodeArg(encodeArg(arg)); got != arg { + t.Errorf("[%d] objabi.DecodeArg(encodeArg(%q)) = %q [seed: %v]", i, arg, got, seed) + } + } +} diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go new file mode 100644 index 0000000..cc4e2b2 --- /dev/null +++ b/src/cmd/go/internal/work/gc.go @@ -0,0 +1,693 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "bufio" + "bytes" + "fmt" + "io" + "log" + "os" + "path/filepath" + "runtime" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/load" + "cmd/go/internal/str" + "cmd/internal/objabi" + "cmd/internal/sys" + "crypto/sha1" +) + +// The 'path' used for GOROOT_FINAL when -trimpath is specified +const trimPathGoRootFinal = "go" + +// The Go toolchain. + +type gcToolchain struct{} + +func (gcToolchain) compiler() string { + return base.Tool("compile") +} + +func (gcToolchain) linker() string { + return base.Tool("link") +} + +func pkgPath(a *Action) string { + p := a.Package + ppath := p.ImportPath + if cfg.BuildBuildmode == "plugin" { + ppath = pluginPath(a) + } else if p.Name == "main" && !p.Internal.ForceLibrary { + ppath = "main" + } + return ppath +} + +func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, output []byte, err error) { + p := a.Package + objdir := a.Objdir + if archive != "" { + ofile = archive + } else { + out := "_go_.o" + ofile = objdir + out + } + + pkgpath := pkgPath(a) + gcargs := []string{"-p", pkgpath} + if p.Module != nil && p.Module.GoVersion != "" && allowedVersion(p.Module.GoVersion) { + gcargs = append(gcargs, "-lang=go"+p.Module.GoVersion) + } + if p.Standard { + gcargs = append(gcargs, "-std") + } + compilingRuntime := p.Standard && (p.ImportPath == "runtime" || strings.HasPrefix(p.ImportPath, "runtime/internal")) + // The runtime package imports a couple of general internal packages. + if p.Standard && (p.ImportPath == "internal/cpu" || p.ImportPath == "internal/bytealg") { + compilingRuntime = true + } + if compilingRuntime { + // runtime compiles with a special gc flag to check for + // memory allocations that are invalid in the runtime package, + // and to implement some special compiler pragmas. + gcargs = append(gcargs, "-+") + } + + // If we're giving the compiler the entire package (no C etc files), tell it that, + // so that it can give good error messages about forward declarations. + // Exceptions: a few standard packages have forward declarations for + // pieces supplied behind-the-scenes by package runtime. + extFiles := len(p.CgoFiles) + len(p.CFiles) + len(p.CXXFiles) + len(p.MFiles) + len(p.FFiles) + len(p.SFiles) + len(p.SysoFiles) + len(p.SwigFiles) + len(p.SwigCXXFiles) + if p.Standard { + switch p.ImportPath { + case "bytes", "internal/poll", "net", "os": + fallthrough + case "runtime/metrics", "runtime/pprof", "runtime/trace": + fallthrough + case "sync", "syscall", "time": + extFiles++ + } + } + if extFiles == 0 { + gcargs = append(gcargs, "-complete") + } + if cfg.BuildContext.InstallSuffix != "" { + gcargs = append(gcargs, "-installsuffix", cfg.BuildContext.InstallSuffix) + } + if a.buildID != "" { + gcargs = append(gcargs, "-buildid", a.buildID) + } + if p.Internal.OmitDebug || cfg.Goos == "plan9" || cfg.Goarch == "wasm" { + gcargs = append(gcargs, "-dwarf=false") + } + if strings.HasPrefix(runtimeVersion, "go1") && !strings.Contains(os.Args[0], "go_bootstrap") { + gcargs = append(gcargs, "-goversion", runtimeVersion) + } + if symabis != "" { + gcargs = append(gcargs, "-symabis", symabis) + } + + gcflags := str.StringList(forcedGcflags, p.Internal.Gcflags) + if compilingRuntime { + // Remove -N, if present. + // It is not possible to build the runtime with no optimizations, + // because the compiler cannot eliminate enough write barriers. + for i := 0; i < len(gcflags); i++ { + if gcflags[i] == "-N" { + copy(gcflags[i:], gcflags[i+1:]) + gcflags = gcflags[:len(gcflags)-1] + i-- + } + } + } + + args := []interface{}{cfg.BuildToolexec, base.Tool("compile"), "-o", ofile, "-trimpath", a.trimpath(), gcflags, gcargs, "-D", p.Internal.LocalPrefix} + if importcfg != nil { + if err := b.writeFile(objdir+"importcfg", importcfg); err != nil { + return "", nil, err + } + args = append(args, "-importcfg", objdir+"importcfg") + } + if embedcfg != nil { + if err := b.writeFile(objdir+"embedcfg", embedcfg); err != nil { + return "", nil, err + } + args = append(args, "-embedcfg", objdir+"embedcfg") + } + if ofile == archive { + args = append(args, "-pack") + } + if asmhdr { + args = append(args, "-asmhdr", objdir+"go_asm.h") + } + + // Add -c=N to use concurrent backend compilation, if possible. + if c := gcBackendConcurrency(gcflags); c > 1 { + args = append(args, fmt.Sprintf("-c=%d", c)) + } + + for _, f := range gofiles { + f := mkAbs(p.Dir, f) + + // Handle overlays. Convert path names using OverlayPath + // so these paths can be handed directly to tools. + // Deleted files won't show up in when scanning directories earlier, + // so OverlayPath will never return "" (meaning a deleted file) here. + // TODO(#39958): Handle cases where the package directory + // doesn't exist on disk (this can happen when all the package's + // files are in an overlay): the code expects the package directory + // to exist and runs some tools in that directory. + // TODO(#39958): Process the overlays when the + // gofiles, cgofiles, cfiles, sfiles, and cxxfiles variables are + // created in (*Builder).build. Doing that requires rewriting the + // code that uses those values to expect absolute paths. + f, _ = fsys.OverlayPath(f) + + args = append(args, f) + } + + output, err = b.runOut(a, base.Cwd, nil, args...) + return ofile, output, err +} + +// gcBackendConcurrency returns the backend compiler concurrency level for a package compilation. +func gcBackendConcurrency(gcflags []string) int { + // First, check whether we can use -c at all for this compilation. + canDashC := concurrentGCBackendCompilationEnabledByDefault + + switch e := os.Getenv("GO19CONCURRENTCOMPILATION"); e { + case "0": + canDashC = false + case "1": + canDashC = true + case "": + // Not set. Use default. + default: + log.Fatalf("GO19CONCURRENTCOMPILATION must be 0, 1, or unset, got %q", e) + } + +CheckFlags: + for _, flag := range gcflags { + // Concurrent compilation is presumed incompatible with any gcflags, + // except for known commonly used flags. + // If the user knows better, they can manually add their own -c to the gcflags. + switch flag { + case "-N", "-l", "-S", "-B", "-C", "-I": + // OK + default: + canDashC = false + break CheckFlags + } + } + + // TODO: Test and delete these conditions. + if objabi.Fieldtrack_enabled != 0 || objabi.Preemptibleloops_enabled != 0 { + canDashC = false + } + + if !canDashC { + return 1 + } + + // Decide how many concurrent backend compilations to allow. + // + // If we allow too many, in theory we might end up with p concurrent processes, + // each with c concurrent backend compiles, all fighting over the same resources. + // However, in practice, that seems not to happen too much. + // Most build graphs are surprisingly serial, so p==1 for much of the build. + // Furthermore, concurrent backend compilation is only enabled for a part + // of the overall compiler execution, so c==1 for much of the build. + // So don't worry too much about that interaction for now. + // + // However, in practice, setting c above 4 tends not to help very much. + // See the analysis in CL 41192. + // + // TODO(josharian): attempt to detect whether this particular compilation + // is likely to be a bottleneck, e.g. when: + // - it has no successor packages to compile (usually package main) + // - all paths through the build graph pass through it + // - critical path scheduling says it is high priority + // and in such a case, set c to runtime.NumCPU. + // We do this now when p==1. + if cfg.BuildP == 1 { + // No process parallelism. Max out c. + return runtime.NumCPU() + } + // Some process parallelism. Set c to min(4, numcpu). + c := 4 + if ncpu := runtime.NumCPU(); ncpu < c { + c = ncpu + } + return c +} + +// trimpath returns the -trimpath argument to use +// when compiling the action. +func (a *Action) trimpath() string { + // Keep in sync with Builder.ccompile + // The trimmed paths are a little different, but we need to trim in the + // same situations. + + // Strip the object directory entirely. + objdir := a.Objdir + if len(objdir) > 1 && objdir[len(objdir)-1] == filepath.Separator { + objdir = objdir[:len(objdir)-1] + } + rewrite := "" + + rewriteDir := a.Package.Dir + if cfg.BuildTrimpath { + if m := a.Package.Module; m != nil && m.Version != "" { + rewriteDir = m.Path + "@" + m.Version + strings.TrimPrefix(a.Package.ImportPath, m.Path) + } else { + rewriteDir = a.Package.ImportPath + } + rewrite += a.Package.Dir + "=>" + rewriteDir + ";" + } + + // Add rewrites for overlays. The 'from' and 'to' paths in overlays don't need to have + // same basename, so go from the overlay contents file path (passed to the compiler) + // to the path the disk path would be rewritten to. + + cgoFiles := make(map[string]bool) + for _, f := range a.Package.CgoFiles { + cgoFiles[f] = true + } + + // TODO(matloob): Higher up in the stack, when the logic for deciding when to make copies + // of c/c++/m/f/hfiles is consolidated, use the same logic that Build uses to determine + // whether to create the copies in objdir to decide whether to rewrite objdir to the + // package directory here. + var overlayNonGoRewrites string // rewrites for non-go files + hasCgoOverlay := false + if fsys.OverlayFile != "" { + for _, filename := range a.Package.AllFiles() { + path := filename + if !filepath.IsAbs(path) { + path = filepath.Join(a.Package.Dir, path) + } + base := filepath.Base(path) + isGo := strings.HasSuffix(filename, ".go") || strings.HasSuffix(filename, ".s") + isCgo := cgoFiles[filename] || !isGo + overlayPath, isOverlay := fsys.OverlayPath(path) + if isCgo && isOverlay { + hasCgoOverlay = true + } + if !isCgo && isOverlay { + rewrite += overlayPath + "=>" + filepath.Join(rewriteDir, base) + ";" + } else if isCgo { + // Generate rewrites for non-Go files copied to files in objdir. + if filepath.Dir(path) == a.Package.Dir { + // This is a file copied to objdir. + overlayNonGoRewrites += filepath.Join(objdir, base) + "=>" + filepath.Join(rewriteDir, base) + ";" + } + } else { + // Non-overlay Go files are covered by the a.Package.Dir rewrite rule above. + } + } + } + if hasCgoOverlay { + rewrite += overlayNonGoRewrites + } + rewrite += objdir + "=>" + + return rewrite +} + +func asmArgs(a *Action, p *load.Package) []interface{} { + // Add -I pkg/GOOS_GOARCH so #include "textflag.h" works in .s files. + inc := filepath.Join(cfg.GOROOT, "pkg", "include") + pkgpath := pkgPath(a) + args := []interface{}{cfg.BuildToolexec, base.Tool("asm"), "-p", pkgpath, "-trimpath", a.trimpath(), "-I", a.Objdir, "-I", inc, "-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch, forcedAsmflags, p.Internal.Asmflags} + if p.ImportPath == "runtime" && cfg.Goarch == "386" { + for _, arg := range forcedAsmflags { + if arg == "-dynlink" { + args = append(args, "-D=GOBUILDMODE_shared=1") + } + } + } + if objabi.IsRuntimePackagePath(pkgpath) { + args = append(args, "-compiling-runtime") + if objabi.Regabi_enabled != 0 { + // In order to make it easier to port runtime assembly + // to the register ABI, we introduce a macro + // indicating the experiment is enabled. + // + // Note: a similar change also appears in + // cmd/dist/build.go. + // + // TODO(austin): Remove this once we commit to the + // register ABI (#40724). + args = append(args, "-D=GOEXPERIMENT_REGABI=1") + } + } + + if cfg.Goarch == "mips" || cfg.Goarch == "mipsle" { + // Define GOMIPS_value from cfg.GOMIPS. + args = append(args, "-D", "GOMIPS_"+cfg.GOMIPS) + } + + if cfg.Goarch == "mips64" || cfg.Goarch == "mips64le" { + // Define GOMIPS64_value from cfg.GOMIPS64. + args = append(args, "-D", "GOMIPS64_"+cfg.GOMIPS64) + } + + return args +} + +func (gcToolchain) asm(b *Builder, a *Action, sfiles []string) ([]string, error) { + p := a.Package + args := asmArgs(a, p) + + var ofiles []string + for _, sfile := range sfiles { + overlayPath, _ := fsys.OverlayPath(mkAbs(p.Dir, sfile)) + ofile := a.Objdir + sfile[:len(sfile)-len(".s")] + ".o" + ofiles = append(ofiles, ofile) + args1 := append(args, "-o", ofile, overlayPath) + if err := b.run(a, p.Dir, p.ImportPath, nil, args1...); err != nil { + return nil, err + } + } + return ofiles, nil +} + +func (gcToolchain) symabis(b *Builder, a *Action, sfiles []string) (string, error) { + mkSymabis := func(p *load.Package, sfiles []string, path string) error { + args := asmArgs(a, p) + args = append(args, "-gensymabis", "-o", path) + for _, sfile := range sfiles { + if p.ImportPath == "runtime/cgo" && strings.HasPrefix(sfile, "gcc_") { + continue + } + op, _ := fsys.OverlayPath(mkAbs(p.Dir, sfile)) + args = append(args, op) + } + + // Supply an empty go_asm.h as if the compiler had been run. + // -gensymabis parsing is lax enough that we don't need the + // actual definitions that would appear in go_asm.h. + if err := b.writeFile(a.Objdir+"go_asm.h", nil); err != nil { + return err + } + + return b.run(a, p.Dir, p.ImportPath, nil, args...) + } + + var symabis string // Only set if we actually create the file + p := a.Package + if len(sfiles) != 0 { + symabis = a.Objdir + "symabis" + if err := mkSymabis(p, sfiles, symabis); err != nil { + return "", err + } + } + + return symabis, nil +} + +// toolVerify checks that the command line args writes the same output file +// if run using newTool instead. +// Unused now but kept around for future use. +func toolVerify(a *Action, b *Builder, p *load.Package, newTool string, ofile string, args []interface{}) error { + newArgs := make([]interface{}, len(args)) + copy(newArgs, args) + newArgs[1] = base.Tool(newTool) + newArgs[3] = ofile + ".new" // x.6 becomes x.6.new + if err := b.run(a, p.Dir, p.ImportPath, nil, newArgs...); err != nil { + return err + } + data1, err := os.ReadFile(ofile) + if err != nil { + return err + } + data2, err := os.ReadFile(ofile + ".new") + if err != nil { + return err + } + if !bytes.Equal(data1, data2) { + return fmt.Errorf("%s and %s produced different output files:\n%s\n%s", filepath.Base(args[1].(string)), newTool, strings.Join(str.StringList(args...), " "), strings.Join(str.StringList(newArgs...), " ")) + } + os.Remove(ofile + ".new") + return nil +} + +func (gcToolchain) pack(b *Builder, a *Action, afile string, ofiles []string) error { + var absOfiles []string + for _, f := range ofiles { + absOfiles = append(absOfiles, mkAbs(a.Objdir, f)) + } + absAfile := mkAbs(a.Objdir, afile) + + // The archive file should have been created by the compiler. + // Since it used to not work that way, verify. + if !cfg.BuildN { + if _, err := os.Stat(absAfile); err != nil { + base.Fatalf("os.Stat of archive file failed: %v", err) + } + } + + p := a.Package + if cfg.BuildN || cfg.BuildX { + cmdline := str.StringList(base.Tool("pack"), "r", absAfile, absOfiles) + b.Showcmd(p.Dir, "%s # internal", joinUnambiguously(cmdline)) + } + if cfg.BuildN { + return nil + } + if err := packInternal(absAfile, absOfiles); err != nil { + b.showOutput(a, p.Dir, p.Desc(), err.Error()+"\n") + return errPrintedOutput + } + return nil +} + +func packInternal(afile string, ofiles []string) error { + dst, err := os.OpenFile(afile, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + return err + } + defer dst.Close() // only for error returns or panics + w := bufio.NewWriter(dst) + + for _, ofile := range ofiles { + src, err := os.Open(ofile) + if err != nil { + return err + } + fi, err := src.Stat() + if err != nil { + src.Close() + return err + } + // Note: Not using %-16.16s format because we care + // about bytes, not runes. + name := fi.Name() + if len(name) > 16 { + name = name[:16] + } else { + name += strings.Repeat(" ", 16-len(name)) + } + size := fi.Size() + fmt.Fprintf(w, "%s%-12d%-6d%-6d%-8o%-10d`\n", + name, 0, 0, 0, 0644, size) + n, err := io.Copy(w, src) + src.Close() + if err == nil && n < size { + err = io.ErrUnexpectedEOF + } else if err == nil && n > size { + err = fmt.Errorf("file larger than size reported by stat") + } + if err != nil { + return fmt.Errorf("copying %s to %s: %v", ofile, afile, err) + } + if size&1 != 0 { + w.WriteByte(0) + } + } + + if err := w.Flush(); err != nil { + return err + } + return dst.Close() +} + +// setextld sets the appropriate linker flags for the specified compiler. +func setextld(ldflags []string, compiler []string) []string { + for _, f := range ldflags { + if f == "-extld" || strings.HasPrefix(f, "-extld=") { + // don't override -extld if supplied + return ldflags + } + } + ldflags = append(ldflags, "-extld="+compiler[0]) + if len(compiler) > 1 { + extldflags := false + add := strings.Join(compiler[1:], " ") + for i, f := range ldflags { + if f == "-extldflags" && i+1 < len(ldflags) { + ldflags[i+1] = add + " " + ldflags[i+1] + extldflags = true + break + } else if strings.HasPrefix(f, "-extldflags=") { + ldflags[i] = "-extldflags=" + add + " " + ldflags[i][len("-extldflags="):] + extldflags = true + break + } + } + if !extldflags { + ldflags = append(ldflags, "-extldflags="+add) + } + } + return ldflags +} + +// pluginPath computes the package path for a plugin main package. +// +// This is typically the import path of the main package p, unless the +// plugin is being built directly from source files. In that case we +// combine the package build ID with the contents of the main package +// source files. This allows us to identify two different plugins +// built from two source files with the same name. +func pluginPath(a *Action) string { + p := a.Package + if p.ImportPath != "command-line-arguments" { + return p.ImportPath + } + h := sha1.New() + buildID := a.buildID + if a.Mode == "link" { + // For linking, use the main package's build ID instead of + // the binary's build ID, so it is the same hash used in + // compiling and linking. + // When compiling, we use actionID/actionID (instead of + // actionID/contentID) as a temporary build ID to compute + // the hash. Do the same here. (See buildid.go:useCache) + // The build ID matters because it affects the overall hash + // in the plugin's pseudo-import path returned below. + // We need to use the same import path when compiling and linking. + id := strings.Split(buildID, buildIDSeparator) + buildID = id[1] + buildIDSeparator + id[1] + } + fmt.Fprintf(h, "build ID: %s\n", buildID) + for _, file := range str.StringList(p.GoFiles, p.CgoFiles, p.SFiles) { + data, err := os.ReadFile(filepath.Join(p.Dir, file)) + if err != nil { + base.Fatalf("go: %s", err) + } + h.Write(data) + } + return fmt.Sprintf("plugin/unnamed-%x", h.Sum(nil)) +} + +func (gcToolchain) ld(b *Builder, root *Action, out, importcfg, mainpkg string) error { + cxx := len(root.Package.CXXFiles) > 0 || len(root.Package.SwigCXXFiles) > 0 + for _, a := range root.Deps { + if a.Package != nil && (len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0) { + cxx = true + } + } + var ldflags []string + if cfg.BuildContext.InstallSuffix != "" { + ldflags = append(ldflags, "-installsuffix", cfg.BuildContext.InstallSuffix) + } + if root.Package.Internal.OmitDebug { + ldflags = append(ldflags, "-s", "-w") + } + if cfg.BuildBuildmode == "plugin" { + ldflags = append(ldflags, "-pluginpath", pluginPath(root)) + } + + // Store BuildID inside toolchain binaries as a unique identifier of the + // tool being run, for use by content-based staleness determination. + if root.Package.Goroot && strings.HasPrefix(root.Package.ImportPath, "cmd/") { + // External linking will include our build id in the external + // linker's build id, which will cause our build id to not + // match the next time the tool is built. + // Rely on the external build id instead. + if !sys.MustLinkExternal(cfg.Goos, cfg.Goarch) { + ldflags = append(ldflags, "-X=cmd/internal/objabi.buildID="+root.buildID) + } + } + + // If the user has not specified the -extld option, then specify the + // appropriate linker. In case of C++ code, use the compiler named + // by the CXX environment variable or defaultCXX if CXX is not set. + // Else, use the CC environment variable and defaultCC as fallback. + var compiler []string + if cxx { + compiler = envList("CXX", cfg.DefaultCXX(cfg.Goos, cfg.Goarch)) + } else { + compiler = envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch)) + } + ldflags = append(ldflags, "-buildmode="+ldBuildmode) + if root.buildID != "" { + ldflags = append(ldflags, "-buildid="+root.buildID) + } + ldflags = append(ldflags, forcedLdflags...) + ldflags = append(ldflags, root.Package.Internal.Ldflags...) + ldflags = setextld(ldflags, compiler) + + // On OS X when using external linking to build a shared library, + // the argument passed here to -o ends up recorded in the final + // shared library in the LC_ID_DYLIB load command. + // To avoid putting the temporary output directory name there + // (and making the resulting shared library useless), + // run the link in the output directory so that -o can name + // just the final path element. + // On Windows, DLL file name is recorded in PE file + // export section, so do like on OS X. + dir := "." + if (cfg.Goos == "darwin" || cfg.Goos == "windows") && cfg.BuildBuildmode == "c-shared" { + dir, out = filepath.Split(out) + } + + env := []string{} + if cfg.BuildTrimpath { + env = append(env, "GOROOT_FINAL="+trimPathGoRootFinal) + } + return b.run(root, dir, root.Package.ImportPath, env, cfg.BuildToolexec, base.Tool("link"), "-o", out, "-importcfg", importcfg, ldflags, mainpkg) +} + +func (gcToolchain) ldShared(b *Builder, root *Action, toplevelactions []*Action, out, importcfg string, allactions []*Action) error { + ldflags := []string{"-installsuffix", cfg.BuildContext.InstallSuffix} + ldflags = append(ldflags, "-buildmode=shared") + ldflags = append(ldflags, forcedLdflags...) + ldflags = append(ldflags, root.Package.Internal.Ldflags...) + cxx := false + for _, a := range allactions { + if a.Package != nil && (len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0) { + cxx = true + } + } + // If the user has not specified the -extld option, then specify the + // appropriate linker. In case of C++ code, use the compiler named + // by the CXX environment variable or defaultCXX if CXX is not set. + // Else, use the CC environment variable and defaultCC as fallback. + var compiler []string + if cxx { + compiler = envList("CXX", cfg.DefaultCXX(cfg.Goos, cfg.Goarch)) + } else { + compiler = envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch)) + } + ldflags = setextld(ldflags, compiler) + for _, d := range toplevelactions { + if !strings.HasSuffix(d.Target, ".a") { // omit unsafe etc and actions for other shared libraries + continue + } + ldflags = append(ldflags, d.Package.ImportPath+"="+d.Target) + } + return b.run(root, ".", out, nil, cfg.BuildToolexec, base.Tool("link"), "-o", out, "-importcfg", importcfg, ldflags) +} + +func (gcToolchain) cc(b *Builder, a *Action, ofile, cfile string) error { + return fmt.Errorf("%s: C source files not supported without cgo", mkAbs(a.Package.Dir, cfile)) +} diff --git a/src/cmd/go/internal/work/gccgo.go b/src/cmd/go/internal/work/gccgo.go new file mode 100644 index 0000000..b58c8aa --- /dev/null +++ b/src/cmd/go/internal/work/gccgo.go @@ -0,0 +1,618 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "fmt" + exec "internal/execabs" + "os" + "path/filepath" + "strings" + "sync" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/load" + "cmd/go/internal/str" + "cmd/internal/pkgpath" +) + +// The Gccgo toolchain. + +type gccgoToolchain struct{} + +var GccgoName, GccgoBin string +var gccgoErr error + +func init() { + GccgoName = cfg.Getenv("GCCGO") + if GccgoName == "" { + GccgoName = "gccgo" + } + GccgoBin, gccgoErr = exec.LookPath(GccgoName) +} + +func (gccgoToolchain) compiler() string { + checkGccgoBin() + return GccgoBin +} + +func (gccgoToolchain) linker() string { + checkGccgoBin() + return GccgoBin +} + +func (gccgoToolchain) ar() string { + ar := cfg.Getenv("AR") + if ar == "" { + ar = "ar" + } + return ar +} + +func checkGccgoBin() { + if gccgoErr == nil { + return + } + fmt.Fprintf(os.Stderr, "cmd/go: gccgo: %s\n", gccgoErr) + base.SetExitStatus(2) + base.Exit() +} + +func (tools gccgoToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, output []byte, err error) { + p := a.Package + objdir := a.Objdir + out := "_go_.o" + ofile = objdir + out + gcargs := []string{"-g"} + gcargs = append(gcargs, b.gccArchArgs()...) + gcargs = append(gcargs, "-fdebug-prefix-map="+b.WorkDir+"=/tmp/go-build") + gcargs = append(gcargs, "-gno-record-gcc-switches") + if pkgpath := gccgoPkgpath(p); pkgpath != "" { + gcargs = append(gcargs, "-fgo-pkgpath="+pkgpath) + } + if p.Internal.LocalPrefix != "" { + gcargs = append(gcargs, "-fgo-relative-import-path="+p.Internal.LocalPrefix) + } + + args := str.StringList(tools.compiler(), "-c", gcargs, "-o", ofile, forcedGccgoflags) + if importcfg != nil { + if b.gccSupportsFlag(args[:1], "-fgo-importcfg=/dev/null") { + if err := b.writeFile(objdir+"importcfg", importcfg); err != nil { + return "", nil, err + } + args = append(args, "-fgo-importcfg="+objdir+"importcfg") + } else { + root := objdir + "_importcfgroot_" + if err := buildImportcfgSymlinks(b, root, importcfg); err != nil { + return "", nil, err + } + args = append(args, "-I", root) + } + } + if embedcfg != nil && b.gccSupportsFlag(args[:1], "-fgo-embedcfg=/dev/null") { + if err := b.writeFile(objdir+"embedcfg", embedcfg); err != nil { + return "", nil, err + } + args = append(args, "-fgo-embedcfg="+objdir+"embedcfg") + } + + if b.gccSupportsFlag(args[:1], "-ffile-prefix-map=a=b") { + if cfg.BuildTrimpath { + args = append(args, "-ffile-prefix-map="+base.Cwd+"=.") + args = append(args, "-ffile-prefix-map="+b.WorkDir+"=/tmp/go-build") + } + if fsys.OverlayFile != "" { + for _, name := range gofiles { + absPath := mkAbs(p.Dir, name) + overlayPath, ok := fsys.OverlayPath(absPath) + if !ok { + continue + } + toPath := absPath + // gccgo only applies the last matching rule, so also handle the case where + // BuildTrimpath is true and the path is relative to base.Cwd. + if cfg.BuildTrimpath && str.HasFilePathPrefix(toPath, base.Cwd) { + toPath = "." + toPath[len(base.Cwd):] + } + args = append(args, "-ffile-prefix-map="+overlayPath+"="+toPath) + } + } + } + + args = append(args, a.Package.Internal.Gccgoflags...) + for _, f := range gofiles { + f := mkAbs(p.Dir, f) + // Overlay files if necessary. + // See comment on gctoolchain.gc about overlay TODOs + f, _ = fsys.OverlayPath(f) + args = append(args, f) + } + + output, err = b.runOut(a, p.Dir, nil, args) + return ofile, output, err +} + +// buildImportcfgSymlinks builds in root a tree of symlinks +// implementing the directives from importcfg. +// This serves as a temporary transition mechanism until +// we can depend on gccgo reading an importcfg directly. +// (The Go 1.9 and later gc compilers already do.) +func buildImportcfgSymlinks(b *Builder, root string, importcfg []byte) error { + for lineNum, line := range strings.Split(string(importcfg), "\n") { + lineNum++ // 1-based + line = strings.TrimSpace(line) + if line == "" { + continue + } + if line == "" || strings.HasPrefix(line, "#") { + continue + } + var verb, args string + if i := strings.Index(line, " "); i < 0 { + verb = line + } else { + verb, args = line[:i], strings.TrimSpace(line[i+1:]) + } + var before, after string + if i := strings.Index(args, "="); i >= 0 { + before, after = args[:i], args[i+1:] + } + switch verb { + default: + base.Fatalf("importcfg:%d: unknown directive %q", lineNum, verb) + case "packagefile": + if before == "" || after == "" { + return fmt.Errorf(`importcfg:%d: invalid packagefile: syntax is "packagefile path=filename": %s`, lineNum, line) + } + archive := gccgoArchive(root, before) + if err := b.Mkdir(filepath.Dir(archive)); err != nil { + return err + } + if err := b.Symlink(after, archive); err != nil { + return err + } + case "importmap": + if before == "" || after == "" { + return fmt.Errorf(`importcfg:%d: invalid importmap: syntax is "importmap old=new": %s`, lineNum, line) + } + beforeA := gccgoArchive(root, before) + afterA := gccgoArchive(root, after) + if err := b.Mkdir(filepath.Dir(beforeA)); err != nil { + return err + } + if err := b.Mkdir(filepath.Dir(afterA)); err != nil { + return err + } + if err := b.Symlink(afterA, beforeA); err != nil { + return err + } + case "packageshlib": + return fmt.Errorf("gccgo -importcfg does not support shared libraries") + } + } + return nil +} + +func (tools gccgoToolchain) asm(b *Builder, a *Action, sfiles []string) ([]string, error) { + p := a.Package + var ofiles []string + for _, sfile := range sfiles { + base := filepath.Base(sfile) + ofile := a.Objdir + base[:len(base)-len(".s")] + ".o" + ofiles = append(ofiles, ofile) + sfile, _ = fsys.OverlayPath(mkAbs(p.Dir, sfile)) + defs := []string{"-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch} + if pkgpath := tools.gccgoCleanPkgpath(b, p); pkgpath != "" { + defs = append(defs, `-D`, `GOPKGPATH=`+pkgpath) + } + defs = tools.maybePIC(defs) + defs = append(defs, b.gccArchArgs()...) + err := b.run(a, p.Dir, p.ImportPath, nil, tools.compiler(), "-xassembler-with-cpp", "-I", a.Objdir, "-c", "-o", ofile, defs, sfile) + if err != nil { + return nil, err + } + } + return ofiles, nil +} + +func (gccgoToolchain) symabis(b *Builder, a *Action, sfiles []string) (string, error) { + return "", nil +} + +func gccgoArchive(basedir, imp string) string { + end := filepath.FromSlash(imp + ".a") + afile := filepath.Join(basedir, end) + // add "lib" to the final element + return filepath.Join(filepath.Dir(afile), "lib"+filepath.Base(afile)) +} + +func (tools gccgoToolchain) pack(b *Builder, a *Action, afile string, ofiles []string) error { + p := a.Package + objdir := a.Objdir + var absOfiles []string + for _, f := range ofiles { + absOfiles = append(absOfiles, mkAbs(objdir, f)) + } + var arArgs []string + if cfg.Goos == "aix" && cfg.Goarch == "ppc64" { + // AIX puts both 32-bit and 64-bit objects in the same archive. + // Tell the AIX "ar" command to only care about 64-bit objects. + arArgs = []string{"-X64"} + } + absAfile := mkAbs(objdir, afile) + // Try with D modifier first, then without if that fails. + output, err := b.runOut(a, p.Dir, nil, tools.ar(), arArgs, "rcD", absAfile, absOfiles) + if err != nil { + return b.run(a, p.Dir, p.ImportPath, nil, tools.ar(), arArgs, "rc", absAfile, absOfiles) + } + + if len(output) > 0 { + // Show the output if there is any even without errors. + b.showOutput(a, p.Dir, p.ImportPath, b.processOutput(output)) + } + + return nil +} + +func (tools gccgoToolchain) link(b *Builder, root *Action, out, importcfg string, allactions []*Action, buildmode, desc string) error { + // gccgo needs explicit linking with all package dependencies, + // and all LDFLAGS from cgo dependencies. + afiles := []string{} + shlibs := []string{} + ldflags := b.gccArchArgs() + cgoldflags := []string{} + usesCgo := false + cxx := false + objc := false + fortran := false + if root.Package != nil { + cxx = len(root.Package.CXXFiles) > 0 || len(root.Package.SwigCXXFiles) > 0 + objc = len(root.Package.MFiles) > 0 + fortran = len(root.Package.FFiles) > 0 + } + + readCgoFlags := func(flagsFile string) error { + flags, err := os.ReadFile(flagsFile) + if err != nil { + return err + } + const ldflagsPrefix = "_CGO_LDFLAGS=" + for _, line := range strings.Split(string(flags), "\n") { + if strings.HasPrefix(line, ldflagsPrefix) { + newFlags := strings.Fields(line[len(ldflagsPrefix):]) + for _, flag := range newFlags { + // Every _cgo_flags file has -g and -O2 in _CGO_LDFLAGS + // but they don't mean anything to the linker so filter + // them out. + if flag != "-g" && !strings.HasPrefix(flag, "-O") { + cgoldflags = append(cgoldflags, flag) + } + } + } + } + return nil + } + + var arArgs []string + if cfg.Goos == "aix" && cfg.Goarch == "ppc64" { + // AIX puts both 32-bit and 64-bit objects in the same archive. + // Tell the AIX "ar" command to only care about 64-bit objects. + arArgs = []string{"-X64"} + } + + newID := 0 + readAndRemoveCgoFlags := func(archive string) (string, error) { + newID++ + newArchive := root.Objdir + fmt.Sprintf("_pkg%d_.a", newID) + if err := b.copyFile(newArchive, archive, 0666, false); err != nil { + return "", err + } + if cfg.BuildN || cfg.BuildX { + b.Showcmd("", "ar d %s _cgo_flags", newArchive) + if cfg.BuildN { + // TODO(rsc): We could do better about showing the right _cgo_flags even in -n mode. + // Either the archive is already built and we can read them out, + // or we're printing commands to build the archive and can + // forward the _cgo_flags directly to this step. + return "", nil + } + } + err := b.run(root, root.Objdir, desc, nil, tools.ar(), arArgs, "x", newArchive, "_cgo_flags") + if err != nil { + return "", err + } + err = b.run(root, ".", desc, nil, tools.ar(), arArgs, "d", newArchive, "_cgo_flags") + if err != nil { + return "", err + } + err = readCgoFlags(filepath.Join(root.Objdir, "_cgo_flags")) + if err != nil { + return "", err + } + return newArchive, nil + } + + // If using -linkshared, find the shared library deps. + haveShlib := make(map[string]bool) + targetBase := filepath.Base(root.Target) + if cfg.BuildLinkshared { + for _, a := range root.Deps { + p := a.Package + if p == nil || p.Shlib == "" { + continue + } + + // The .a we are linking into this .so + // will have its Shlib set to this .so. + // Don't start thinking we want to link + // this .so into itself. + base := filepath.Base(p.Shlib) + if base != targetBase { + haveShlib[base] = true + } + } + } + + // Arrange the deps into afiles and shlibs. + addedShlib := make(map[string]bool) + for _, a := range root.Deps { + p := a.Package + if p != nil && p.Shlib != "" && haveShlib[filepath.Base(p.Shlib)] { + // This is a package linked into a shared + // library that we will put into shlibs. + continue + } + + if haveShlib[filepath.Base(a.Target)] { + // This is a shared library we want to link against. + if !addedShlib[a.Target] { + shlibs = append(shlibs, a.Target) + addedShlib[a.Target] = true + } + continue + } + + if p != nil { + target := a.built + if p.UsesCgo() || p.UsesSwig() { + var err error + target, err = readAndRemoveCgoFlags(target) + if err != nil { + continue + } + } + + afiles = append(afiles, target) + } + } + + for _, a := range allactions { + // Gather CgoLDFLAGS, but not from standard packages. + // The go tool can dig up runtime/cgo from GOROOT and + // think that it should use its CgoLDFLAGS, but gccgo + // doesn't use runtime/cgo. + if a.Package == nil { + continue + } + if !a.Package.Standard { + cgoldflags = append(cgoldflags, a.Package.CgoLDFLAGS...) + } + if len(a.Package.CgoFiles) > 0 { + usesCgo = true + } + if a.Package.UsesSwig() { + usesCgo = true + } + if len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0 { + cxx = true + } + if len(a.Package.MFiles) > 0 { + objc = true + } + if len(a.Package.FFiles) > 0 { + fortran = true + } + } + + wholeArchive := []string{"-Wl,--whole-archive"} + noWholeArchive := []string{"-Wl,--no-whole-archive"} + if cfg.Goos == "aix" { + wholeArchive = nil + noWholeArchive = nil + } + ldflags = append(ldflags, wholeArchive...) + ldflags = append(ldflags, afiles...) + ldflags = append(ldflags, noWholeArchive...) + + ldflags = append(ldflags, cgoldflags...) + ldflags = append(ldflags, envList("CGO_LDFLAGS", "")...) + if root.Package != nil { + ldflags = append(ldflags, root.Package.CgoLDFLAGS...) + } + if cfg.Goos != "aix" { + ldflags = str.StringList("-Wl,-(", ldflags, "-Wl,-)") + } + + if root.buildID != "" { + // On systems that normally use gold or the GNU linker, + // use the --build-id option to write a GNU build ID note. + switch cfg.Goos { + case "android", "dragonfly", "linux", "netbsd": + ldflags = append(ldflags, fmt.Sprintf("-Wl,--build-id=0x%x", root.buildID)) + } + } + + var rLibPath string + if cfg.Goos == "aix" { + rLibPath = "-Wl,-blibpath=" + } else { + rLibPath = "-Wl,-rpath=" + } + for _, shlib := range shlibs { + ldflags = append( + ldflags, + "-L"+filepath.Dir(shlib), + rLibPath+filepath.Dir(shlib), + "-l"+strings.TrimSuffix( + strings.TrimPrefix(filepath.Base(shlib), "lib"), + ".so")) + } + + var realOut string + goLibBegin := str.StringList(wholeArchive, "-lgolibbegin", noWholeArchive) + switch buildmode { + case "exe": + if usesCgo && cfg.Goos == "linux" { + ldflags = append(ldflags, "-Wl,-E") + } + + case "c-archive": + // Link the Go files into a single .o, and also link + // in -lgolibbegin. + // + // We need to use --whole-archive with -lgolibbegin + // because it doesn't define any symbols that will + // cause the contents to be pulled in; it's just + // initialization code. + // + // The user remains responsible for linking against + // -lgo -lpthread -lm in the final link. We can't use + // -r to pick them up because we can't combine + // split-stack and non-split-stack code in a single -r + // link, and libgo picks up non-split-stack code from + // libffi. + ldflags = append(ldflags, "-Wl,-r", "-nostdlib") + ldflags = append(ldflags, goLibBegin...) + + if nopie := b.gccNoPie([]string{tools.linker()}); nopie != "" { + ldflags = append(ldflags, nopie) + } + + // We are creating an object file, so we don't want a build ID. + if root.buildID == "" { + ldflags = b.disableBuildID(ldflags) + } + + realOut = out + out = out + ".o" + + case "c-shared": + ldflags = append(ldflags, "-shared", "-nostdlib") + ldflags = append(ldflags, goLibBegin...) + ldflags = append(ldflags, "-lgo", "-lgcc_s", "-lgcc", "-lc", "-lgcc") + + case "shared": + if cfg.Goos != "aix" { + ldflags = append(ldflags, "-zdefs") + } + ldflags = append(ldflags, "-shared", "-nostdlib", "-lgo", "-lgcc_s", "-lgcc", "-lc") + + default: + base.Fatalf("-buildmode=%s not supported for gccgo", buildmode) + } + + switch buildmode { + case "exe", "c-shared": + if cxx { + ldflags = append(ldflags, "-lstdc++") + } + if objc { + ldflags = append(ldflags, "-lobjc") + } + if fortran { + fc := cfg.Getenv("FC") + if fc == "" { + fc = "gfortran" + } + // support gfortran out of the box and let others pass the correct link options + // via CGO_LDFLAGS + if strings.Contains(fc, "gfortran") { + ldflags = append(ldflags, "-lgfortran") + } + } + } + + if err := b.run(root, ".", desc, nil, tools.linker(), "-o", out, ldflags, forcedGccgoflags, root.Package.Internal.Gccgoflags); err != nil { + return err + } + + switch buildmode { + case "c-archive": + if err := b.run(root, ".", desc, nil, tools.ar(), arArgs, "rc", realOut, out); err != nil { + return err + } + } + return nil +} + +func (tools gccgoToolchain) ld(b *Builder, root *Action, out, importcfg, mainpkg string) error { + return tools.link(b, root, out, importcfg, root.Deps, ldBuildmode, root.Package.ImportPath) +} + +func (tools gccgoToolchain) ldShared(b *Builder, root *Action, toplevelactions []*Action, out, importcfg string, allactions []*Action) error { + return tools.link(b, root, out, importcfg, allactions, "shared", out) +} + +func (tools gccgoToolchain) cc(b *Builder, a *Action, ofile, cfile string) error { + p := a.Package + inc := filepath.Join(cfg.GOROOT, "pkg", "include") + cfile = mkAbs(p.Dir, cfile) + defs := []string{"-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch} + defs = append(defs, b.gccArchArgs()...) + if pkgpath := tools.gccgoCleanPkgpath(b, p); pkgpath != "" { + defs = append(defs, `-D`, `GOPKGPATH="`+pkgpath+`"`) + } + compiler := envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch)) + if b.gccSupportsFlag(compiler, "-fsplit-stack") { + defs = append(defs, "-fsplit-stack") + } + defs = tools.maybePIC(defs) + if b.gccSupportsFlag(compiler, "-ffile-prefix-map=a=b") { + defs = append(defs, "-ffile-prefix-map="+base.Cwd+"=.") + defs = append(defs, "-ffile-prefix-map="+b.WorkDir+"=/tmp/go-build") + } else if b.gccSupportsFlag(compiler, "-fdebug-prefix-map=a=b") { + defs = append(defs, "-fdebug-prefix-map="+b.WorkDir+"=/tmp/go-build") + } + if b.gccSupportsFlag(compiler, "-gno-record-gcc-switches") { + defs = append(defs, "-gno-record-gcc-switches") + } + return b.run(a, p.Dir, p.ImportPath, nil, compiler, "-Wall", "-g", + "-I", a.Objdir, "-I", inc, "-o", ofile, defs, "-c", cfile) +} + +// maybePIC adds -fPIC to the list of arguments if needed. +func (tools gccgoToolchain) maybePIC(args []string) []string { + switch cfg.BuildBuildmode { + case "c-shared", "shared", "plugin": + args = append(args, "-fPIC") + } + return args +} + +func gccgoPkgpath(p *load.Package) string { + if p.Internal.Build.IsCommand() && !p.Internal.ForceLibrary { + return "" + } + return p.ImportPath +} + +var gccgoToSymbolFuncOnce sync.Once +var gccgoToSymbolFunc func(string) string + +func (tools gccgoToolchain) gccgoCleanPkgpath(b *Builder, p *load.Package) string { + gccgoToSymbolFuncOnce.Do(func() { + fn, err := pkgpath.ToSymbolFunc(tools.compiler(), b.WorkDir) + if err != nil { + fmt.Fprintf(os.Stderr, "cmd/go: %v\n", err) + base.SetExitStatus(2) + base.Exit() + } + gccgoToSymbolFunc = fn + }) + + return gccgoToSymbolFunc(gccgoPkgpath(p)) +} diff --git a/src/cmd/go/internal/work/init.go b/src/cmd/go/internal/work/init.go new file mode 100644 index 0000000..ba7c7c2 --- /dev/null +++ b/src/cmd/go/internal/work/init.go @@ -0,0 +1,286 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Build initialization (after flag parsing). + +package work + +import ( + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/modload" + "cmd/internal/objabi" + "cmd/internal/sys" + "flag" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" +) + +func BuildInit() { + modload.Init() + instrumentInit() + buildModeInit() + if err := fsys.Init(base.Cwd); err != nil { + base.Fatalf("go: %v", err) + } + + // Make sure -pkgdir is absolute, because we run commands + // in different directories. + if cfg.BuildPkgdir != "" && !filepath.IsAbs(cfg.BuildPkgdir) { + p, err := filepath.Abs(cfg.BuildPkgdir) + if err != nil { + fmt.Fprintf(os.Stderr, "go %s: evaluating -pkgdir: %v\n", flag.Args()[0], err) + base.SetExitStatus(2) + base.Exit() + } + cfg.BuildPkgdir = p + } + + // Make sure CC and CXX are absolute paths + for _, key := range []string{"CC", "CXX"} { + if path := cfg.Getenv(key); !filepath.IsAbs(path) && path != "" && path != filepath.Base(path) { + base.Fatalf("go %s: %s environment variable is relative; must be absolute path: %s\n", flag.Args()[0], key, path) + } + } + + // For each experiment that has been enabled in the toolchain, define a + // build tag with the same name but prefixed by "goexperiment." which can be + // used for compiling alternative files for the experiment. This allows + // changes for the experiment, like extra struct fields in the runtime, + // without affecting the base non-experiment code at all. [2:] strips the + // leading "X:" from objabi.Expstring(). + exp := objabi.Expstring()[2:] + if exp != "none" { + experiments := strings.Split(exp, ",") + for _, expt := range experiments { + cfg.BuildContext.BuildTags = append(cfg.BuildContext.BuildTags, "goexperiment."+expt) + } + } +} + +func instrumentInit() { + if !cfg.BuildRace && !cfg.BuildMSan { + return + } + if cfg.BuildRace && cfg.BuildMSan { + fmt.Fprintf(os.Stderr, "go %s: may not use -race and -msan simultaneously\n", flag.Args()[0]) + base.SetExitStatus(2) + base.Exit() + } + if cfg.BuildMSan && !sys.MSanSupported(cfg.Goos, cfg.Goarch) { + fmt.Fprintf(os.Stderr, "-msan is not supported on %s/%s\n", cfg.Goos, cfg.Goarch) + base.SetExitStatus(2) + base.Exit() + } + if cfg.BuildRace { + if !sys.RaceDetectorSupported(cfg.Goos, cfg.Goarch) { + fmt.Fprintf(os.Stderr, "go %s: -race is only supported on linux/amd64, linux/ppc64le, linux/arm64, freebsd/amd64, netbsd/amd64, darwin/amd64, darwin/arm64, and windows/amd64\n", flag.Args()[0]) + base.SetExitStatus(2) + base.Exit() + } + } + mode := "race" + if cfg.BuildMSan { + mode = "msan" + // MSAN does not support non-PIE binaries on ARM64. + // See issue #33712 for details. + if cfg.Goos == "linux" && cfg.Goarch == "arm64" && cfg.BuildBuildmode == "default" { + cfg.BuildBuildmode = "pie" + } + } + modeFlag := "-" + mode + + if !cfg.BuildContext.CgoEnabled { + if runtime.GOOS != cfg.Goos || runtime.GOARCH != cfg.Goarch { + fmt.Fprintf(os.Stderr, "go %s: %s requires cgo\n", flag.Args()[0], modeFlag) + } else { + fmt.Fprintf(os.Stderr, "go %s: %s requires cgo; enable cgo by setting CGO_ENABLED=1\n", flag.Args()[0], modeFlag) + } + + base.SetExitStatus(2) + base.Exit() + } + forcedGcflags = append(forcedGcflags, modeFlag) + forcedLdflags = append(forcedLdflags, modeFlag) + + if cfg.BuildContext.InstallSuffix != "" { + cfg.BuildContext.InstallSuffix += "_" + } + cfg.BuildContext.InstallSuffix += mode + cfg.BuildContext.BuildTags = append(cfg.BuildContext.BuildTags, mode) +} + +func buildModeInit() { + gccgo := cfg.BuildToolchainName == "gccgo" + var codegenArg string + + // Configure the build mode first, then verify that it is supported. + // That way, if the flag is completely bogus we will prefer to error out with + // "-buildmode=%s not supported" instead of naming the specific platform. + + switch cfg.BuildBuildmode { + case "archive": + pkgsFilter = pkgsNotMain + case "c-archive": + pkgsFilter = oneMainPkg + if gccgo { + codegenArg = "-fPIC" + } else { + switch cfg.Goos { + case "darwin", "ios": + switch cfg.Goarch { + case "arm64": + codegenArg = "-shared" + } + + case "dragonfly", "freebsd", "illumos", "linux", "netbsd", "openbsd", "solaris": + // Use -shared so that the result is + // suitable for inclusion in a PIE or + // shared library. + codegenArg = "-shared" + } + } + cfg.ExeSuffix = ".a" + ldBuildmode = "c-archive" + case "c-shared": + pkgsFilter = oneMainPkg + if gccgo { + codegenArg = "-fPIC" + } else { + switch cfg.Goos { + case "linux", "android", "freebsd": + codegenArg = "-shared" + case "windows": + // Do not add usual .exe suffix to the .dll file. + cfg.ExeSuffix = "" + } + } + ldBuildmode = "c-shared" + case "default": + switch cfg.Goos { + case "android": + codegenArg = "-shared" + ldBuildmode = "pie" + case "windows": + ldBuildmode = "pie" + case "ios": + codegenArg = "-shared" + ldBuildmode = "pie" + case "darwin": + switch cfg.Goarch { + case "arm64": + codegenArg = "-shared" + } + fallthrough + default: + ldBuildmode = "exe" + } + if gccgo { + codegenArg = "" + } + case "exe": + pkgsFilter = pkgsMain + ldBuildmode = "exe" + // Set the pkgsFilter to oneMainPkg if the user passed a specific binary output + // and is using buildmode=exe for a better error message. + // See issue #20017. + if cfg.BuildO != "" { + pkgsFilter = oneMainPkg + } + case "pie": + if cfg.BuildRace { + base.Fatalf("-buildmode=pie not supported when -race is enabled") + } + if gccgo { + codegenArg = "-fPIE" + } else { + switch cfg.Goos { + case "aix", "windows": + default: + codegenArg = "-shared" + } + } + ldBuildmode = "pie" + case "shared": + pkgsFilter = pkgsNotMain + if gccgo { + codegenArg = "-fPIC" + } else { + codegenArg = "-dynlink" + } + if cfg.BuildO != "" { + base.Fatalf("-buildmode=shared and -o not supported together") + } + ldBuildmode = "shared" + case "plugin": + pkgsFilter = oneMainPkg + if gccgo { + codegenArg = "-fPIC" + } else { + codegenArg = "-dynlink" + } + cfg.ExeSuffix = ".so" + ldBuildmode = "plugin" + default: + base.Fatalf("buildmode=%s not supported", cfg.BuildBuildmode) + } + + if !sys.BuildModeSupported(cfg.BuildToolchainName, cfg.BuildBuildmode, cfg.Goos, cfg.Goarch) { + base.Fatalf("-buildmode=%s not supported on %s/%s\n", cfg.BuildBuildmode, cfg.Goos, cfg.Goarch) + } + + if cfg.BuildLinkshared { + if !sys.BuildModeSupported(cfg.BuildToolchainName, "shared", cfg.Goos, cfg.Goarch) { + base.Fatalf("-linkshared not supported on %s/%s\n", cfg.Goos, cfg.Goarch) + } + if gccgo { + codegenArg = "-fPIC" + } else { + forcedAsmflags = append(forcedAsmflags, "-D=GOBUILDMODE_shared=1", + "-linkshared") + codegenArg = "-dynlink" + forcedGcflags = append(forcedGcflags, "-linkshared") + // TODO(mwhudson): remove -w when that gets fixed in linker. + forcedLdflags = append(forcedLdflags, "-linkshared", "-w") + } + } + if codegenArg != "" { + if gccgo { + forcedGccgoflags = append([]string{codegenArg}, forcedGccgoflags...) + } else { + forcedAsmflags = append([]string{codegenArg}, forcedAsmflags...) + forcedGcflags = append([]string{codegenArg}, forcedGcflags...) + } + // Don't alter InstallSuffix when modifying default codegen args. + if cfg.BuildBuildmode != "default" || cfg.BuildLinkshared { + if cfg.BuildContext.InstallSuffix != "" { + cfg.BuildContext.InstallSuffix += "_" + } + cfg.BuildContext.InstallSuffix += codegenArg[1:] + } + } + + switch cfg.BuildMod { + case "": + // Behavior will be determined automatically, as if no flag were passed. + case "readonly", "vendor", "mod": + if !cfg.ModulesEnabled && !base.InGOFLAGS("-mod") { + base.Fatalf("build flag -mod=%s only valid when using modules", cfg.BuildMod) + } + default: + base.Fatalf("-mod=%s not supported (can be '', 'mod', 'readonly', or 'vendor')", cfg.BuildMod) + } + if !cfg.ModulesEnabled { + if cfg.ModCacheRW && !base.InGOFLAGS("-modcacherw") { + base.Fatalf("build flag -modcacherw only valid when using modules") + } + if cfg.ModFile != "" && !base.InGOFLAGS("-mod") { + base.Fatalf("build flag -modfile only valid when using modules") + } + } +} diff --git a/src/cmd/go/internal/work/security.go b/src/cmd/go/internal/work/security.go new file mode 100644 index 0000000..36bbab3 --- /dev/null +++ b/src/cmd/go/internal/work/security.go @@ -0,0 +1,310 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Checking of compiler and linker flags. +// We must avoid flags like -fplugin=, which can allow +// arbitrary code execution during the build. +// Do not make changes here without carefully +// considering the implications. +// (That's why the code is isolated in a file named security.go.) +// +// Note that -Wl,foo means split foo on commas and pass to +// the linker, so that -Wl,-foo,bar means pass -foo bar to +// the linker. Similarly -Wa,foo for the assembler and so on. +// If any of these are permitted, the wildcard portion must +// disallow commas. +// +// Note also that GNU binutils accept any argument @foo +// as meaning "read more flags from the file foo", so we must +// guard against any command-line argument beginning with @, +// even things like "-I @foo". +// We use load.SafeArg (which is even more conservative) +// to reject these. +// +// Even worse, gcc -I@foo (one arg) turns into cc1 -I @foo (two args), +// so although gcc doesn't expand the @foo, cc1 will. +// So out of paranoia, we reject @ at the beginning of every +// flag argument that might be split into its own argument. + +package work + +import ( + "fmt" + "internal/lazyregexp" + "regexp" + "strings" + + "cmd/go/internal/cfg" + "cmd/go/internal/load" +) + +var re = lazyregexp.New + +var validCompilerFlags = []*lazyregexp.Regexp{ + re(`-D([A-Za-z_][A-Za-z0-9_]*)(=[^@\-]*)?`), + re(`-U([A-Za-z_][A-Za-z0-9_]*)`), + re(`-F([^@\-].*)`), + re(`-I([^@\-].*)`), + re(`-O`), + re(`-O([^@\-].*)`), + re(`-W`), + re(`-W([^@,]+)`), // -Wall but not -Wa,-foo. + re(`-Wa,-mbig-obj`), + re(`-Wp,-D([A-Za-z_][A-Za-z0-9_]*)(=[^@,\-]*)?`), + re(`-Wp,-U([A-Za-z_][A-Za-z0-9_]*)`), + re(`-ansi`), + re(`-f(no-)?asynchronous-unwind-tables`), + re(`-f(no-)?blocks`), + re(`-f(no-)builtin-[a-zA-Z0-9_]*`), + re(`-f(no-)?common`), + re(`-f(no-)?constant-cfstrings`), + re(`-fdiagnostics-show-note-include-stack`), + re(`-f(no-)?eliminate-unused-debug-types`), + re(`-f(no-)?exceptions`), + re(`-f(no-)?fast-math`), + re(`-f(no-)?inline-functions`), + re(`-finput-charset=([^@\-].*)`), + re(`-f(no-)?fat-lto-objects`), + re(`-f(no-)?keep-inline-dllexport`), + re(`-f(no-)?lto`), + re(`-fmacro-backtrace-limit=(.+)`), + re(`-fmessage-length=(.+)`), + re(`-f(no-)?modules`), + re(`-f(no-)?objc-arc`), + re(`-f(no-)?objc-nonfragile-abi`), + re(`-f(no-)?objc-legacy-dispatch`), + re(`-f(no-)?omit-frame-pointer`), + re(`-f(no-)?openmp(-simd)?`), + re(`-f(no-)?permissive`), + re(`-f(no-)?(pic|PIC|pie|PIE)`), + re(`-f(no-)?plt`), + re(`-f(no-)?rtti`), + re(`-f(no-)?split-stack`), + re(`-f(no-)?stack-(.+)`), + re(`-f(no-)?strict-aliasing`), + re(`-f(un)signed-char`), + re(`-f(no-)?use-linker-plugin`), // safe if -B is not used; we don't permit -B + re(`-f(no-)?visibility-inlines-hidden`), + re(`-fsanitize=(.+)`), + re(`-ftemplate-depth-(.+)`), + re(`-fvisibility=(.+)`), + re(`-g([^@\-].*)?`), + re(`-m32`), + re(`-m64`), + re(`-m(abi|arch|cpu|fpu|tune)=([^@\-].*)`), + re(`-m(no-)?v?aes`), + re(`-marm`), + re(`-m(no-)?avx[0-9a-z]*`), + re(`-mfloat-abi=([^@\-].*)`), + re(`-mfpmath=[0-9a-z,+]*`), + re(`-m(no-)?avx[0-9a-z.]*`), + re(`-m(no-)?ms-bitfields`), + re(`-m(no-)?stack-(.+)`), + re(`-mmacosx-(.+)`), + re(`-mios-simulator-version-min=(.+)`), + re(`-miphoneos-version-min=(.+)`), + re(`-mtvos-simulator-version-min=(.+)`), + re(`-mtvos-version-min=(.+)`), + re(`-mwatchos-simulator-version-min=(.+)`), + re(`-mwatchos-version-min=(.+)`), + re(`-mnop-fun-dllimport`), + re(`-m(no-)?sse[0-9.]*`), + re(`-m(no-)?ssse3`), + re(`-mthumb(-interwork)?`), + re(`-mthreads`), + re(`-mwindows`), + re(`--param=ssp-buffer-size=[0-9]*`), + re(`-pedantic(-errors)?`), + re(`-pipe`), + re(`-pthread`), + re(`-?-std=([^@\-].*)`), + re(`-?-stdlib=([^@\-].*)`), + re(`--sysroot=([^@\-].*)`), + re(`-w`), + re(`-x([^@\-].*)`), + re(`-v`), +} + +var validCompilerFlagsWithNextArg = []string{ + "-arch", + "-D", + "-U", + "-I", + "-framework", + "-include", + "-isysroot", + "-isystem", + "--sysroot", + "-target", + "-x", +} + +var validLinkerFlags = []*lazyregexp.Regexp{ + re(`-F([^@\-].*)`), + re(`-l([^@\-].*)`), + re(`-L([^@\-].*)`), + re(`-O`), + re(`-O([^@\-].*)`), + re(`-f(no-)?(pic|PIC|pie|PIE)`), + re(`-f(no-)?openmp(-simd)?`), + re(`-fsanitize=([^@\-].*)`), + re(`-flat_namespace`), + re(`-g([^@\-].*)?`), + re(`-headerpad_max_install_names`), + re(`-m(abi|arch|cpu|fpu|tune)=([^@\-].*)`), + re(`-mfloat-abi=([^@\-].*)`), + re(`-mmacosx-(.+)`), + re(`-mios-simulator-version-min=(.+)`), + re(`-miphoneos-version-min=(.+)`), + re(`-mthreads`), + re(`-mwindows`), + re(`-(pic|PIC|pie|PIE)`), + re(`-pthread`), + re(`-rdynamic`), + re(`-shared`), + re(`-?-static([-a-z0-9+]*)`), + re(`-?-stdlib=([^@\-].*)`), + re(`-v`), + + // Note that any wildcards in -Wl need to exclude comma, + // since -Wl splits its argument at commas and passes + // them all to the linker uninterpreted. Allowing comma + // in a wildcard would allow tunnelling arbitrary additional + // linker arguments through one of these. + re(`-Wl,--(no-)?allow-multiple-definition`), + re(`-Wl,--(no-)?allow-shlib-undefined`), + re(`-Wl,--(no-)?as-needed`), + re(`-Wl,-Bdynamic`), + re(`-Wl,-berok`), + re(`-Wl,-Bstatic`), + re(`-Wl,-Bsymbolic-functions`), + re(`-Wl,-O([^@,\-][^,]*)?`), + re(`-Wl,-d[ny]`), + re(`-Wl,--disable-new-dtags`), + re(`-Wl,-e[=,][a-zA-Z0-9]*`), + re(`-Wl,--enable-new-dtags`), + re(`-Wl,--end-group`), + re(`-Wl,--(no-)?export-dynamic`), + re(`-Wl,-E`), + re(`-Wl,-framework,[^,@\-][^,]+`), + re(`-Wl,--hash-style=(sysv|gnu|both)`), + re(`-Wl,-headerpad_max_install_names`), + re(`-Wl,--no-undefined`), + re(`-Wl,-R([^@\-][^,@]*$)`), + re(`-Wl,--just-symbols[=,]([^,@\-][^,@]+)`), + re(`-Wl,-rpath(-link)?[=,]([^,@\-][^,]+)`), + re(`-Wl,-s`), + re(`-Wl,-search_paths_first`), + re(`-Wl,-sectcreate,([^,@\-][^,]+),([^,@\-][^,]+),([^,@\-][^,]+)`), + re(`-Wl,--start-group`), + re(`-Wl,-?-static`), + re(`-Wl,-?-subsystem,(native|windows|console|posix|xbox)`), + re(`-Wl,-syslibroot[=,]([^,@\-][^,]+)`), + re(`-Wl,-undefined[=,]([^,@\-][^,]+)`), + re(`-Wl,-?-unresolved-symbols=[^,]+`), + re(`-Wl,--(no-)?warn-([^,]+)`), + re(`-Wl,-?-wrap[=,][^,@\-][^,]*`), + re(`-Wl,-z,(no)?execstack`), + re(`-Wl,-z,relro`), + + re(`[a-zA-Z0-9_/].*\.(a|o|obj|dll|dylib|so)`), // direct linker inputs: x.o or libfoo.so (but not -foo.o or @foo.o) + re(`\./.*\.(a|o|obj|dll|dylib|so)`), +} + +var validLinkerFlagsWithNextArg = []string{ + "-arch", + "-F", + "-l", + "-L", + "-framework", + "-isysroot", + "--sysroot", + "-target", + "-Wl,-framework", + "-Wl,-rpath", + "-Wl,-R", + "-Wl,--just-symbols", + "-Wl,-undefined", +} + +func checkCompilerFlags(name, source string, list []string) error { + return checkFlags(name, source, list, validCompilerFlags, validCompilerFlagsWithNextArg) +} + +func checkLinkerFlags(name, source string, list []string) error { + return checkFlags(name, source, list, validLinkerFlags, validLinkerFlagsWithNextArg) +} + +func checkFlags(name, source string, list []string, valid []*lazyregexp.Regexp, validNext []string) error { + // Let users override rules with $CGO_CFLAGS_ALLOW, $CGO_CFLAGS_DISALLOW, etc. + var ( + allow *regexp.Regexp + disallow *regexp.Regexp + ) + if env := cfg.Getenv("CGO_" + name + "_ALLOW"); env != "" { + r, err := regexp.Compile(env) + if err != nil { + return fmt.Errorf("parsing $CGO_%s_ALLOW: %v", name, err) + } + allow = r + } + if env := cfg.Getenv("CGO_" + name + "_DISALLOW"); env != "" { + r, err := regexp.Compile(env) + if err != nil { + return fmt.Errorf("parsing $CGO_%s_DISALLOW: %v", name, err) + } + disallow = r + } + +Args: + for i := 0; i < len(list); i++ { + arg := list[i] + if disallow != nil && disallow.FindString(arg) == arg { + goto Bad + } + if allow != nil && allow.FindString(arg) == arg { + continue Args + } + for _, re := range valid { + if re.FindString(arg) == arg { // must be complete match + continue Args + } + } + for _, x := range validNext { + if arg == x { + if i+1 < len(list) && load.SafeArg(list[i+1]) { + i++ + continue Args + } + + // Permit -Wl,-framework -Wl,name. + if i+1 < len(list) && + strings.HasPrefix(arg, "-Wl,") && + strings.HasPrefix(list[i+1], "-Wl,") && + load.SafeArg(list[i+1][4:]) && + !strings.Contains(list[i+1][4:], ",") { + i++ + continue Args + } + + // Permit -I= /path, -I $SYSROOT. + if i+1 < len(list) && arg == "-I" { + if (strings.HasPrefix(list[i+1], "=") || strings.HasPrefix(list[i+1], "$SYSROOT")) && + load.SafeArg(list[i+1][1:]) { + i++ + continue Args + } + } + + if i+1 < len(list) { + return fmt.Errorf("invalid flag in %s: %s %s (see https://golang.org/s/invalidflag)", source, arg, list[i+1]) + } + return fmt.Errorf("invalid flag in %s: %s without argument (see https://golang.org/s/invalidflag)", source, arg) + } + } + Bad: + return fmt.Errorf("invalid flag in %s: %s", source, arg) + } + return nil +} diff --git a/src/cmd/go/internal/work/security_test.go b/src/cmd/go/internal/work/security_test.go new file mode 100644 index 0000000..4f2e0eb --- /dev/null +++ b/src/cmd/go/internal/work/security_test.go @@ -0,0 +1,277 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "os" + "testing" +) + +var goodCompilerFlags = [][]string{ + {"-DFOO"}, + {"-Dfoo=bar"}, + {"-Ufoo"}, + {"-Ufoo1"}, + {"-F/Qt"}, + {"-I/"}, + {"-I/etc/passwd"}, + {"-I."}, + {"-O"}, + {"-O2"}, + {"-Osmall"}, + {"-W"}, + {"-Wall"}, + {"-Wp,-Dfoo=bar"}, + {"-Wp,-Ufoo"}, + {"-Wp,-Dfoo1"}, + {"-Wp,-Ufoo1"}, + {"-fobjc-arc"}, + {"-fno-objc-arc"}, + {"-fomit-frame-pointer"}, + {"-fno-omit-frame-pointer"}, + {"-fpic"}, + {"-fno-pic"}, + {"-fPIC"}, + {"-fno-PIC"}, + {"-fpie"}, + {"-fno-pie"}, + {"-fPIE"}, + {"-fno-PIE"}, + {"-fsplit-stack"}, + {"-fno-split-stack"}, + {"-fstack-xxx"}, + {"-fno-stack-xxx"}, + {"-fsanitize=hands"}, + {"-g"}, + {"-ggdb"}, + {"-march=souza"}, + {"-mcpu=123"}, + {"-mfpu=123"}, + {"-mtune=happybirthday"}, + {"-mstack-overflow"}, + {"-mno-stack-overflow"}, + {"-mmacosx-version"}, + {"-mnop-fun-dllimport"}, + {"-pthread"}, + {"-std=c99"}, + {"-xc"}, + {"-D", "FOO"}, + {"-D", "foo=bar"}, + {"-I", "."}, + {"-I", "/etc/passwd"}, + {"-I", "世界"}, + {"-I", "=/usr/include/libxml2"}, + {"-I", "dir"}, + {"-I", "$SYSROOT/dir"}, + {"-isystem", "/usr/include/mozjs-68"}, + {"-include", "/usr/include/mozjs-68/RequiredDefines.h"}, + {"-framework", "Chocolate"}, + {"-x", "c"}, + {"-v"}, +} + +var badCompilerFlags = [][]string{ + {"-D@X"}, + {"-D-X"}, + {"-Ufoo=bar"}, + {"-F@dir"}, + {"-F-dir"}, + {"-I@dir"}, + {"-I-dir"}, + {"-O@1"}, + {"-Wa,-foo"}, + {"-W@foo"}, + {"-Wp,-DX,-D@X"}, + {"-Wp,-UX,-U@X"}, + {"-g@gdb"}, + {"-g-gdb"}, + {"-march=@dawn"}, + {"-march=-dawn"}, + {"-std=@c99"}, + {"-std=-c99"}, + {"-x@c"}, + {"-x-c"}, + {"-D", "@foo"}, + {"-D", "-foo"}, + {"-I", "@foo"}, + {"-I", "-foo"}, + {"-I", "=@obj"}, + {"-include", "@foo"}, + {"-framework", "-Caffeine"}, + {"-framework", "@Home"}, + {"-x", "--c"}, + {"-x", "@obj"}, +} + +func TestCheckCompilerFlags(t *testing.T) { + for _, f := range goodCompilerFlags { + if err := checkCompilerFlags("test", "test", f); err != nil { + t.Errorf("unexpected error for %q: %v", f, err) + } + } + for _, f := range badCompilerFlags { + if err := checkCompilerFlags("test", "test", f); err == nil { + t.Errorf("missing error for %q", f) + } + } +} + +var goodLinkerFlags = [][]string{ + {"-Fbar"}, + {"-lbar"}, + {"-Lbar"}, + {"-fpic"}, + {"-fno-pic"}, + {"-fPIC"}, + {"-fno-PIC"}, + {"-fpie"}, + {"-fno-pie"}, + {"-fPIE"}, + {"-fno-PIE"}, + {"-fsanitize=hands"}, + {"-g"}, + {"-ggdb"}, + {"-march=souza"}, + {"-mcpu=123"}, + {"-mfpu=123"}, + {"-mtune=happybirthday"}, + {"-pic"}, + {"-pthread"}, + {"-Wl,--hash-style=both"}, + {"-Wl,-rpath,foo"}, + {"-Wl,-rpath,$ORIGIN/foo"}, + {"-Wl,-R", "/foo"}, + {"-Wl,-R", "foo"}, + {"-Wl,-R,foo"}, + {"-Wl,--just-symbols=foo"}, + {"-Wl,--just-symbols,foo"}, + {"-Wl,--warn-error"}, + {"-Wl,--no-warn-error"}, + {"foo.so"}, + {"_世界.dll"}, + {"./x.o"}, + {"libcgosotest.dylib"}, + {"-F", "framework"}, + {"-l", "."}, + {"-l", "/etc/passwd"}, + {"-l", "世界"}, + {"-L", "framework"}, + {"-framework", "Chocolate"}, + {"-v"}, + {"-Wl,-sectcreate,__TEXT,__info_plist,${SRCDIR}/Info.plist"}, + {"-Wl,-framework", "-Wl,Chocolate"}, + {"-Wl,-framework,Chocolate"}, + {"-Wl,-unresolved-symbols=ignore-all"}, +} + +var badLinkerFlags = [][]string{ + {"-DFOO"}, + {"-Dfoo=bar"}, + {"-W"}, + {"-Wall"}, + {"-fobjc-arc"}, + {"-fno-objc-arc"}, + {"-fomit-frame-pointer"}, + {"-fno-omit-frame-pointer"}, + {"-fsplit-stack"}, + {"-fno-split-stack"}, + {"-fstack-xxx"}, + {"-fno-stack-xxx"}, + {"-mstack-overflow"}, + {"-mno-stack-overflow"}, + {"-mnop-fun-dllimport"}, + {"-std=c99"}, + {"-xc"}, + {"-D", "FOO"}, + {"-D", "foo=bar"}, + {"-I", "FOO"}, + {"-L", "@foo"}, + {"-L", "-foo"}, + {"-x", "c"}, + {"-D@X"}, + {"-D-X"}, + {"-I@dir"}, + {"-I-dir"}, + {"-O@1"}, + {"-Wa,-foo"}, + {"-W@foo"}, + {"-g@gdb"}, + {"-g-gdb"}, + {"-march=@dawn"}, + {"-march=-dawn"}, + {"-std=@c99"}, + {"-std=-c99"}, + {"-x@c"}, + {"-x-c"}, + {"-D", "@foo"}, + {"-D", "-foo"}, + {"-I", "@foo"}, + {"-I", "-foo"}, + {"-l", "@foo"}, + {"-l", "-foo"}, + {"-framework", "-Caffeine"}, + {"-framework", "@Home"}, + {"-Wl,-framework,-Caffeine"}, + {"-Wl,-framework", "-Wl,@Home"}, + {"-Wl,-framework", "@Home"}, + {"-Wl,-framework,Chocolate,@Home"}, + {"-Wl,--hash-style=foo"}, + {"-x", "--c"}, + {"-x", "@obj"}, + {"-Wl,-rpath,@foo"}, + {"-Wl,-R,foo,bar"}, + {"-Wl,-R,@foo"}, + {"-Wl,--just-symbols,@foo"}, + {"../x.o"}, +} + +func TestCheckLinkerFlags(t *testing.T) { + for _, f := range goodLinkerFlags { + if err := checkLinkerFlags("test", "test", f); err != nil { + t.Errorf("unexpected error for %q: %v", f, err) + } + } + for _, f := range badLinkerFlags { + if err := checkLinkerFlags("test", "test", f); err == nil { + t.Errorf("missing error for %q", f) + } + } +} + +func TestCheckFlagAllowDisallow(t *testing.T) { + if err := checkCompilerFlags("TEST", "test", []string{"-disallow"}); err == nil { + t.Fatalf("missing error for -disallow") + } + os.Setenv("CGO_TEST_ALLOW", "-disallo") + if err := checkCompilerFlags("TEST", "test", []string{"-disallow"}); err == nil { + t.Fatalf("missing error for -disallow with CGO_TEST_ALLOW=-disallo") + } + os.Setenv("CGO_TEST_ALLOW", "-disallow") + if err := checkCompilerFlags("TEST", "test", []string{"-disallow"}); err != nil { + t.Fatalf("unexpected error for -disallow with CGO_TEST_ALLOW=-disallow: %v", err) + } + os.Unsetenv("CGO_TEST_ALLOW") + + if err := checkCompilerFlags("TEST", "test", []string{"-Wall"}); err != nil { + t.Fatalf("unexpected error for -Wall: %v", err) + } + os.Setenv("CGO_TEST_DISALLOW", "-Wall") + if err := checkCompilerFlags("TEST", "test", []string{"-Wall"}); err == nil { + t.Fatalf("missing error for -Wall with CGO_TEST_DISALLOW=-Wall") + } + os.Setenv("CGO_TEST_ALLOW", "-Wall") // disallow wins + if err := checkCompilerFlags("TEST", "test", []string{"-Wall"}); err == nil { + t.Fatalf("missing error for -Wall with CGO_TEST_DISALLOW=-Wall and CGO_TEST_ALLOW=-Wall") + } + + os.Setenv("CGO_TEST_ALLOW", "-fplugin.*") + os.Setenv("CGO_TEST_DISALLOW", "-fplugin=lint.so") + if err := checkCompilerFlags("TEST", "test", []string{"-fplugin=faster.so"}); err != nil { + t.Fatalf("unexpected error for -fplugin=faster.so: %v", err) + } + if err := checkCompilerFlags("TEST", "test", []string{"-fplugin=lint.so"}); err == nil { + t.Fatalf("missing error for -fplugin=lint.so: %v", err) + } +} diff --git a/src/cmd/go/internal/work/testgo.go b/src/cmd/go/internal/work/testgo.go new file mode 100644 index 0000000..931f49a --- /dev/null +++ b/src/cmd/go/internal/work/testgo.go @@ -0,0 +1,48 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains extra hooks for testing the go command. + +// +build testgo + +package work + +import ( + "cmd/go/internal/cfg" + "cmd/go/internal/search" + "fmt" + "os" + "path/filepath" + "runtime" +) + +func init() { + if v := os.Getenv("TESTGO_VERSION"); v != "" { + runtimeVersion = v + } + + if testGOROOT := os.Getenv("TESTGO_GOROOT"); testGOROOT != "" { + // Disallow installs to the GOROOT from which testgo was built. + // Installs to other GOROOTs — such as one set explicitly within a test — are ok. + allowInstall = func(a *Action) error { + if cfg.BuildN { + return nil + } + + rel := search.InDir(a.Target, testGOROOT) + if rel == "" { + return nil + } + + callerPos := "" + if _, file, line, ok := runtime.Caller(1); ok { + if shortFile := search.InDir(file, filepath.Join(testGOROOT, "src")); shortFile != "" { + file = shortFile + } + callerPos = fmt.Sprintf("%s:%d: ", file, line) + } + return fmt.Errorf("%stestgo must not write to GOROOT (installing to %s)", callerPos, filepath.Join("GOROOT", rel)) + } + } +} -- cgit v1.2.3