diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-16 19:25:22 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-16 19:25:22 +0000 |
commit | f6ad4dcef54c5ce997a4bad5a6d86de229015700 (patch) | |
tree | 7cfa4e31ace5c2bd95c72b154d15af494b2bcbef /src/cmd/go/internal/work | |
parent | Initial commit. (diff) | |
download | golang-1.22-f6ad4dcef54c5ce997a4bad5a6d86de229015700.tar.xz golang-1.22-f6ad4dcef54c5ce997a4bad5a6d86de229015700.zip |
Adding upstream version 1.22.1.upstream/1.22.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/cmd/go/internal/work')
-rw-r--r-- | src/cmd/go/internal/work/action.go | 945 | ||||
-rw-r--r-- | src/cmd/go/internal/work/build.go | 961 | ||||
-rw-r--r-- | src/cmd/go/internal/work/build_test.go | 281 | ||||
-rw-r--r-- | src/cmd/go/internal/work/buildid.go | 715 | ||||
-rw-r--r-- | src/cmd/go/internal/work/cover.go | 150 | ||||
-rw-r--r-- | src/cmd/go/internal/work/exec.go | 3472 | ||||
-rw-r--r-- | src/cmd/go/internal/work/exec_test.go | 87 | ||||
-rw-r--r-- | src/cmd/go/internal/work/gc.go | 708 | ||||
-rw-r--r-- | src/cmd/go/internal/work/gccgo.go | 672 | ||||
-rw-r--r-- | src/cmd/go/internal/work/init.go | 424 | ||||
-rw-r--r-- | src/cmd/go/internal/work/security.go | 338 | ||||
-rw-r--r-- | src/cmd/go/internal/work/security_test.go | 318 | ||||
-rw-r--r-- | src/cmd/go/internal/work/shell.go | 678 | ||||
-rw-r--r-- | src/cmd/go/internal/work/shell_test.go | 139 |
14 files changed, 9888 insertions, 0 deletions
diff --git a/src/cmd/go/internal/work/action.go b/src/cmd/go/internal/work/action.go new file mode 100644 index 0000000..a59072e --- /dev/null +++ b/src/cmd/go/internal/work/action.go @@ -0,0 +1,945 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Action graph creation (planning). + +package work + +import ( + "bufio" + "bytes" + "cmd/internal/cov/covcmd" + "container/heap" + "context" + "debug/elf" + "encoding/json" + "fmt" + "internal/platform" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "cmd/go/internal/base" + "cmd/go/internal/cache" + "cmd/go/internal/cfg" + "cmd/go/internal/load" + "cmd/go/internal/robustio" + "cmd/go/internal/str" + "cmd/go/internal/trace" + "cmd/internal/buildid" +) + +// A Builder holds global state about a build. +// It does not hold per-package state, because we +// build packages in parallel, and the builder is shared. +type Builder struct { + WorkDir string // the temporary work directory (ends in filepath.Separator) + actionCache map[cacheKey]*Action // a cache of already-constructed actions + flagCache map[[2]string]bool // a cache of supported compiler flags + gccCompilerIDCache map[string]cache.ActionID // cache for gccCompilerID + + IsCmdList bool // running as part of go list; set p.Stale and additional fields below + NeedError bool // list needs p.Error + NeedExport bool // list needs p.Export + NeedCompiledGoFiles bool // list needs p.CompiledGoFiles + AllowErrors bool // errors don't immediately exit the program + + objdirSeq int // counter for NewObjdir + pkgSeq int + + backgroundSh *Shell // Shell that per-Action Shells are derived from + + exec sync.Mutex + readySema chan bool + ready actionQueue + + id sync.Mutex + toolIDCache map[string]string // tool name -> tool ID + buildIDCache map[string]string // file name -> build ID +} + +// NOTE: Much of Action would not need to be exported if not for test. +// Maybe test functionality should move into this package too? + +// An Actor runs an action. +type Actor interface { + Act(*Builder, context.Context, *Action) error +} + +// An ActorFunc is an Actor that calls the function. +type ActorFunc func(*Builder, context.Context, *Action) error + +func (f ActorFunc) Act(b *Builder, ctx context.Context, a *Action) error { + return f(b, ctx, a) +} + +// An Action represents a single action in the action graph. +type Action struct { + Mode string // description of action operation + Package *load.Package // the package this action works on + Deps []*Action // actions that must happen before this one + Actor Actor // the action itself (nil = no-op) + IgnoreFail bool // whether to run f even if dependencies fail + TestOutput *bytes.Buffer // test output buffer + Args []string // additional args for runProgram + + triggers []*Action // inverse of deps + + buggyInstall bool // is this a buggy install (see -linkshared)? + + TryCache func(*Builder, *Action) bool // callback for cache bypass + + // Generated files, directories. + Objdir string // directory for intermediate objects + Target string // goal of the action: the created package or executable + built string // the actual created package or executable + actionID cache.ActionID // cache ID of action input + buildID string // build ID of action output + + VetxOnly bool // Mode=="vet": only being called to supply info about dependencies + needVet bool // Mode=="build": need to fill in vet config + needBuild bool // Mode=="build": need to do actual build (can be false if needVet is true) + vetCfg *vetConfig // vet config + output []byte // output redirect buffer (nil means use b.Print) + + sh *Shell // lazily created per-Action shell; see Builder.Shell + + // Execution state. + pending int // number of deps yet to complete + priority int // relative execution priority + Failed bool // whether the action failed + json *actionJSON // action graph information + nonGoOverlay map[string]string // map from non-.go source files to copied files in objdir. Nil if no overlay is used. + traceSpan *trace.Span +} + +// BuildActionID returns the action ID section of a's build ID. +func (a *Action) BuildActionID() string { return actionID(a.buildID) } + +// BuildContentID returns the content ID section of a's build ID. +func (a *Action) BuildContentID() string { return contentID(a.buildID) } + +// BuildID returns a's build ID. +func (a *Action) BuildID() string { return a.buildID } + +// BuiltTarget returns the actual file that was built. This differs +// from Target when the result was cached. +func (a *Action) BuiltTarget() string { return a.built } + +// An actionQueue is a priority queue of actions. +type actionQueue []*Action + +// Implement heap.Interface +func (q *actionQueue) Len() int { return len(*q) } +func (q *actionQueue) Swap(i, j int) { (*q)[i], (*q)[j] = (*q)[j], (*q)[i] } +func (q *actionQueue) Less(i, j int) bool { return (*q)[i].priority < (*q)[j].priority } +func (q *actionQueue) Push(x any) { *q = append(*q, x.(*Action)) } +func (q *actionQueue) Pop() any { + n := len(*q) - 1 + x := (*q)[n] + *q = (*q)[:n] + return x +} + +func (q *actionQueue) push(a *Action) { + if a.json != nil { + a.json.TimeReady = time.Now() + } + heap.Push(q, a) +} + +func (q *actionQueue) pop() *Action { + return heap.Pop(q).(*Action) +} + +type actionJSON struct { + ID int + Mode string + Package string + Deps []int `json:",omitempty"` + IgnoreFail bool `json:",omitempty"` + Args []string `json:",omitempty"` + Link bool `json:",omitempty"` + Objdir string `json:",omitempty"` + Target string `json:",omitempty"` + Priority int `json:",omitempty"` + Failed bool `json:",omitempty"` + Built string `json:",omitempty"` + VetxOnly bool `json:",omitempty"` + NeedVet bool `json:",omitempty"` + NeedBuild bool `json:",omitempty"` + ActionID string `json:",omitempty"` + BuildID string `json:",omitempty"` + TimeReady time.Time `json:",omitempty"` + TimeStart time.Time `json:",omitempty"` + TimeDone time.Time `json:",omitempty"` + + Cmd []string // `json:",omitempty"` + CmdReal time.Duration `json:",omitempty"` + CmdUser time.Duration `json:",omitempty"` + CmdSys time.Duration `json:",omitempty"` +} + +// cacheKey is the key for the action cache. +type cacheKey struct { + mode string + p *load.Package +} + +func actionGraphJSON(a *Action) string { + var workq []*Action + var inWorkq = make(map[*Action]int) + + add := func(a *Action) { + if _, ok := inWorkq[a]; ok { + return + } + inWorkq[a] = len(workq) + workq = append(workq, a) + } + add(a) + + for i := 0; i < len(workq); i++ { + for _, dep := range workq[i].Deps { + add(dep) + } + } + + var list []*actionJSON + for id, a := range workq { + if a.json == nil { + a.json = &actionJSON{ + Mode: a.Mode, + ID: id, + IgnoreFail: a.IgnoreFail, + Args: a.Args, + Objdir: a.Objdir, + Target: a.Target, + Failed: a.Failed, + Priority: a.priority, + Built: a.built, + VetxOnly: a.VetxOnly, + NeedBuild: a.needBuild, + NeedVet: a.needVet, + } + if a.Package != nil { + // TODO(rsc): Make this a unique key for a.Package somehow. + a.json.Package = a.Package.ImportPath + } + for _, a1 := range a.Deps { + a.json.Deps = append(a.json.Deps, inWorkq[a1]) + } + } + list = append(list, a.json) + } + + js, err := json.MarshalIndent(list, "", "\t") + if err != nil { + fmt.Fprintf(os.Stderr, "go: writing debug action graph: %v\n", err) + return "" + } + return string(js) +} + +// BuildMode specifies the build mode: +// are we just building things or also installing the results? +type BuildMode int + +const ( + ModeBuild BuildMode = iota + ModeInstall + ModeBuggyInstall + + ModeVetOnly = 1 << 8 +) + +// NewBuilder returns a new Builder ready for use. +// +// If workDir is the empty string, NewBuilder creates a WorkDir if needed +// and arranges for it to be removed in case of an unclean exit. +// The caller must Close the builder explicitly to clean up the WorkDir +// before a clean exit. +func NewBuilder(workDir string) *Builder { + b := new(Builder) + + b.actionCache = make(map[cacheKey]*Action) + b.toolIDCache = make(map[string]string) + b.buildIDCache = make(map[string]string) + + if workDir != "" { + b.WorkDir = workDir + } else if cfg.BuildN { + b.WorkDir = "$WORK" + } else { + if !buildInitStarted { + panic("internal error: NewBuilder called before BuildInit") + } + tmp, err := os.MkdirTemp(cfg.Getenv("GOTMPDIR"), "go-build") + if err != nil { + base.Fatalf("go: creating work dir: %v", err) + } + if !filepath.IsAbs(tmp) { + abs, err := filepath.Abs(tmp) + if err != nil { + os.RemoveAll(tmp) + base.Fatalf("go: creating work dir: %v", err) + } + tmp = abs + } + b.WorkDir = tmp + builderWorkDirs.Store(b, b.WorkDir) + if cfg.BuildX || cfg.BuildWork { + fmt.Fprintf(os.Stderr, "WORK=%s\n", b.WorkDir) + } + } + + b.backgroundSh = NewShell(b.WorkDir, nil) + + if err := CheckGOOSARCHPair(cfg.Goos, cfg.Goarch); err != nil { + fmt.Fprintf(os.Stderr, "go: %v\n", err) + base.SetExitStatus(2) + base.Exit() + } + + for _, tag := range cfg.BuildContext.BuildTags { + if strings.Contains(tag, ",") { + fmt.Fprintf(os.Stderr, "go: -tags space-separated list contains comma\n") + base.SetExitStatus(2) + base.Exit() + } + } + + return b +} + +var builderWorkDirs sync.Map // *Builder → WorkDir + +func (b *Builder) Close() error { + wd, ok := builderWorkDirs.Load(b) + if !ok { + return nil + } + defer builderWorkDirs.Delete(b) + + if b.WorkDir != wd.(string) { + base.Errorf("go: internal error: Builder WorkDir unexpectedly changed from %s to %s", wd, b.WorkDir) + } + + if !cfg.BuildWork { + if err := robustio.RemoveAll(b.WorkDir); err != nil { + return err + } + } + b.WorkDir = "" + return nil +} + +func closeBuilders() { + leakedBuilders := 0 + builderWorkDirs.Range(func(bi, _ any) bool { + leakedBuilders++ + if err := bi.(*Builder).Close(); err != nil { + base.Error(err) + } + return true + }) + + if leakedBuilders > 0 && base.GetExitStatus() == 0 { + fmt.Fprintf(os.Stderr, "go: internal error: Builder leaked on successful exit\n") + base.SetExitStatus(1) + } +} + +func CheckGOOSARCHPair(goos, goarch string) error { + if !platform.BuildModeSupported(cfg.BuildContext.Compiler, "default", goos, goarch) { + return fmt.Errorf("unsupported GOOS/GOARCH pair %s/%s", goos, goarch) + } + return nil +} + +// NewObjdir returns the name of a fresh object directory under b.WorkDir. +// It is up to the caller to call b.Mkdir on the result at an appropriate time. +// The result ends in a slash, so that file names in that directory +// can be constructed with direct string addition. +// +// NewObjdir must be called only from a single goroutine at a time, +// so it is safe to call during action graph construction, but it must not +// be called during action graph execution. +func (b *Builder) NewObjdir() string { + b.objdirSeq++ + return str.WithFilePathSeparator(filepath.Join(b.WorkDir, fmt.Sprintf("b%03d", b.objdirSeq))) +} + +// readpkglist returns the list of packages that were built into the shared library +// at shlibpath. For the native toolchain this list is stored, newline separated, in +// an ELF note with name "Go\x00\x00" and type 1. For GCCGO it is extracted from the +// .go_export section. +func readpkglist(shlibpath string) (pkgs []*load.Package) { + var stk load.ImportStack + if cfg.BuildToolchainName == "gccgo" { + f, err := elf.Open(shlibpath) + if err != nil { + base.Fatal(fmt.Errorf("failed to open shared library: %v", err)) + } + sect := f.Section(".go_export") + if sect == nil { + base.Fatal(fmt.Errorf("%s: missing .go_export section", shlibpath)) + } + data, err := sect.Data() + if err != nil { + base.Fatal(fmt.Errorf("%s: failed to read .go_export section: %v", shlibpath, err)) + } + pkgpath := []byte("pkgpath ") + for _, line := range bytes.Split(data, []byte{'\n'}) { + if path, found := bytes.CutPrefix(line, pkgpath); found { + path = bytes.TrimSuffix(path, []byte{';'}) + pkgs = append(pkgs, load.LoadPackageWithFlags(string(path), base.Cwd(), &stk, nil, 0)) + } + } + } else { + pkglistbytes, err := buildid.ReadELFNote(shlibpath, "Go\x00\x00", 1) + if err != nil { + base.Fatalf("readELFNote failed: %v", err) + } + scanner := bufio.NewScanner(bytes.NewBuffer(pkglistbytes)) + for scanner.Scan() { + t := scanner.Text() + pkgs = append(pkgs, load.LoadPackageWithFlags(t, base.Cwd(), &stk, nil, 0)) + } + } + return +} + +// cacheAction looks up {mode, p} in the cache and returns the resulting action. +// If the cache has no such action, f() is recorded and returned. +// TODO(rsc): Change the second key from *load.Package to interface{}, +// to make the caching in linkShared less awkward? +func (b *Builder) cacheAction(mode string, p *load.Package, f func() *Action) *Action { + a := b.actionCache[cacheKey{mode, p}] + if a == nil { + a = f() + b.actionCache[cacheKey{mode, p}] = a + } + return a +} + +// AutoAction returns the "right" action for go build or go install of p. +func (b *Builder) AutoAction(mode, depMode BuildMode, p *load.Package) *Action { + if p.Name == "main" { + return b.LinkAction(mode, depMode, p) + } + return b.CompileAction(mode, depMode, p) +} + +// buildActor implements the Actor interface for package build +// actions. For most package builds this simply means invoking th +// *Builder.build method; in the case of "go test -cover" for +// a package with no test files, we stores some additional state +// information in the build actor to help with reporting. +type buildActor struct { + // name of static meta-data file fragment emitted by the cover + // tool as part of the package build action, for selected + // "go test -cover" runs. + covMetaFileName string +} + +// newBuildActor returns a new buildActor object, setting up the +// covMetaFileName field if 'genCoverMeta' flag is set. +func newBuildActor(p *load.Package, genCoverMeta bool) *buildActor { + ba := &buildActor{} + if genCoverMeta { + ba.covMetaFileName = covcmd.MetaFileForPackage(p.ImportPath) + } + return ba +} + +func (ba *buildActor) Act(b *Builder, ctx context.Context, a *Action) error { + return b.build(ctx, a) +} + +// CompileAction returns the action for compiling and possibly installing +// (according to mode) the given package. The resulting action is only +// for building packages (archives), never for linking executables. +// depMode is the action (build or install) to use when building dependencies. +// To turn package main into an executable, call b.Link instead. +func (b *Builder) CompileAction(mode, depMode BuildMode, p *load.Package) *Action { + vetOnly := mode&ModeVetOnly != 0 + mode &^= ModeVetOnly + + if mode != ModeBuild && p.Target == "" { + // No permanent target. + mode = ModeBuild + } + if mode != ModeBuild && p.Name == "main" { + // We never install the .a file for a main package. + mode = ModeBuild + } + + // Construct package build action. + a := b.cacheAction("build", p, func() *Action { + a := &Action{ + Mode: "build", + Package: p, + Actor: newBuildActor(p, p.Internal.Cover.GenMeta), + Objdir: b.NewObjdir(), + } + + if p.Error == nil || !p.Error.IsImportCycle { + for _, p1 := range p.Internal.Imports { + a.Deps = append(a.Deps, b.CompileAction(depMode, depMode, p1)) + } + } + + if p.Standard { + switch p.ImportPath { + case "builtin", "unsafe": + // Fake packages - nothing to build. + a.Mode = "built-in package" + a.Actor = nil + return a + } + + // gccgo standard library is "fake" too. + if cfg.BuildToolchainName == "gccgo" { + // the target name is needed for cgo. + a.Mode = "gccgo stdlib" + a.Target = p.Target + a.Actor = nil + return a + } + } + + return a + }) + + // Find the build action; the cache entry may have been replaced + // by the install action during (*Builder).installAction. + buildAction := a + switch buildAction.Mode { + case "build", "built-in package", "gccgo stdlib": + // ok + case "build-install": + buildAction = a.Deps[0] + default: + panic("lost build action: " + buildAction.Mode) + } + buildAction.needBuild = buildAction.needBuild || !vetOnly + + // Construct install action. + if mode == ModeInstall || mode == ModeBuggyInstall { + a = b.installAction(a, mode) + } + + return a +} + +// VetAction returns the action for running go vet on package p. +// It depends on the action for compiling p. +// If the caller may be causing p to be installed, it is up to the caller +// to make sure that the install depends on (runs after) vet. +func (b *Builder) VetAction(mode, depMode BuildMode, p *load.Package) *Action { + a := b.vetAction(mode, depMode, p) + a.VetxOnly = false + return a +} + +func (b *Builder) vetAction(mode, depMode BuildMode, p *load.Package) *Action { + // Construct vet action. + a := b.cacheAction("vet", p, func() *Action { + a1 := b.CompileAction(mode|ModeVetOnly, depMode, p) + + // vet expects to be able to import "fmt". + var stk load.ImportStack + stk.Push("vet") + p1, err := load.LoadImportWithFlags("fmt", p.Dir, p, &stk, nil, 0) + if err != nil { + base.Fatalf("unexpected error loading fmt package from package %s: %v", p.ImportPath, err) + } + stk.Pop() + aFmt := b.CompileAction(ModeBuild, depMode, p1) + + var deps []*Action + if a1.buggyInstall { + // (*Builder).vet expects deps[0] to be the package + // and deps[1] to be "fmt". If we see buggyInstall + // here then a1 is an install of a shared library, + // and the real package is a1.Deps[0]. + deps = []*Action{a1.Deps[0], aFmt, a1} + } else { + deps = []*Action{a1, aFmt} + } + for _, p1 := range p.Internal.Imports { + deps = append(deps, b.vetAction(mode, depMode, p1)) + } + + a := &Action{ + Mode: "vet", + Package: p, + Deps: deps, + Objdir: a1.Objdir, + VetxOnly: true, + IgnoreFail: true, // it's OK if vet of dependencies "fails" (reports problems) + } + if a1.Actor == nil { + // Built-in packages like unsafe. + return a + } + deps[0].needVet = true + a.Actor = ActorFunc((*Builder).vet) + return a + }) + return a +} + +// LinkAction returns the action for linking p into an executable +// and possibly installing the result (according to mode). +// depMode is the action (build or install) to use when compiling dependencies. +func (b *Builder) LinkAction(mode, depMode BuildMode, p *load.Package) *Action { + // Construct link action. + a := b.cacheAction("link", p, func() *Action { + a := &Action{ + Mode: "link", + Package: p, + } + + a1 := b.CompileAction(ModeBuild, depMode, p) + a.Actor = ActorFunc((*Builder).link) + a.Deps = []*Action{a1} + a.Objdir = a1.Objdir + + // An executable file. (This is the name of a temporary file.) + // Because we run the temporary file in 'go run' and 'go test', + // the name will show up in ps listings. If the caller has specified + // a name, use that instead of a.out. The binary is generated + // in an otherwise empty subdirectory named exe to avoid + // naming conflicts. The only possible conflict is if we were + // to create a top-level package named exe. + name := "a.out" + if p.Internal.ExeName != "" { + name = p.Internal.ExeName + } else if (cfg.Goos == "darwin" || cfg.Goos == "windows") && cfg.BuildBuildmode == "c-shared" && p.Target != "" { + // On OS X, the linker output name gets recorded in the + // shared library's LC_ID_DYLIB load command. + // The code invoking the linker knows to pass only the final + // path element. Arrange that the path element matches what + // we'll install it as; otherwise the library is only loadable as "a.out". + // On Windows, DLL file name is recorded in PE file + // export section, so do like on OS X. + _, name = filepath.Split(p.Target) + } + a.Target = a.Objdir + filepath.Join("exe", name) + cfg.ExeSuffix + a.built = a.Target + b.addTransitiveLinkDeps(a, a1, "") + + // Sequence the build of the main package (a1) strictly after the build + // of all other dependencies that go into the link. It is likely to be after + // them anyway, but just make sure. This is required by the build ID-based + // shortcut in (*Builder).useCache(a1), which will call b.linkActionID(a). + // In order for that linkActionID call to compute the right action ID, all the + // dependencies of a (except a1) must have completed building and have + // recorded their build IDs. + a1.Deps = append(a1.Deps, &Action{Mode: "nop", Deps: a.Deps[1:]}) + return a + }) + + if mode == ModeInstall || mode == ModeBuggyInstall { + a = b.installAction(a, mode) + } + + return a +} + +// installAction returns the action for installing the result of a1. +func (b *Builder) installAction(a1 *Action, mode BuildMode) *Action { + // Because we overwrite the build action with the install action below, + // a1 may already be an install action fetched from the "build" cache key, + // and the caller just doesn't realize. + if strings.HasSuffix(a1.Mode, "-install") { + if a1.buggyInstall && mode == ModeInstall { + // Congratulations! The buggy install is now a proper install. + a1.buggyInstall = false + } + return a1 + } + + // If there's no actual action to build a1, + // there's nothing to install either. + // This happens if a1 corresponds to reusing an already-built object. + if a1.Actor == nil { + return a1 + } + + p := a1.Package + return b.cacheAction(a1.Mode+"-install", p, func() *Action { + // The install deletes the temporary build result, + // so we need all other actions, both past and future, + // that attempt to depend on the build to depend instead + // on the install. + + // Make a private copy of a1 (the build action), + // no longer accessible to any other rules. + buildAction := new(Action) + *buildAction = *a1 + + // Overwrite a1 with the install action. + // This takes care of updating past actions that + // point at a1 for the build action; now they will + // point at a1 and get the install action. + // We also leave a1 in the action cache as the result + // for "build", so that actions not yet created that + // try to depend on the build will instead depend + // on the install. + *a1 = Action{ + Mode: buildAction.Mode + "-install", + Actor: ActorFunc(BuildInstallFunc), + Package: p, + Objdir: buildAction.Objdir, + Deps: []*Action{buildAction}, + Target: p.Target, + built: p.Target, + + buggyInstall: mode == ModeBuggyInstall, + } + + b.addInstallHeaderAction(a1) + return a1 + }) +} + +// addTransitiveLinkDeps adds to the link action a all packages +// that are transitive dependencies of a1.Deps. +// That is, if a is a link of package main, a1 is the compile of package main +// and a1.Deps is the actions for building packages directly imported by +// package main (what the compiler needs). The linker needs all packages +// transitively imported by the whole program; addTransitiveLinkDeps +// makes sure those are present in a.Deps. +// If shlib is non-empty, then a corresponds to the build and installation of shlib, +// so any rebuild of shlib should not be added as a dependency. +func (b *Builder) addTransitiveLinkDeps(a, a1 *Action, shlib string) { + // Expand Deps to include all built packages, for the linker. + // Use breadth-first search to find rebuilt-for-test packages + // before the standard ones. + // TODO(rsc): Eliminate the standard ones from the action graph, + // which will require doing a little bit more rebuilding. + workq := []*Action{a1} + haveDep := map[string]bool{} + if a1.Package != nil { + haveDep[a1.Package.ImportPath] = true + } + for i := 0; i < len(workq); i++ { + a1 := workq[i] + for _, a2 := range a1.Deps { + // TODO(rsc): Find a better discriminator than the Mode strings, once the dust settles. + if a2.Package == nil || (a2.Mode != "build-install" && a2.Mode != "build") || haveDep[a2.Package.ImportPath] { + continue + } + haveDep[a2.Package.ImportPath] = true + a.Deps = append(a.Deps, a2) + if a2.Mode == "build-install" { + a2 = a2.Deps[0] // walk children of "build" action + } + workq = append(workq, a2) + } + } + + // If this is go build -linkshared, then the link depends on the shared libraries + // in addition to the packages themselves. (The compile steps do not.) + if cfg.BuildLinkshared { + haveShlib := map[string]bool{shlib: true} + for _, a1 := range a.Deps { + p1 := a1.Package + if p1 == nil || p1.Shlib == "" || haveShlib[filepath.Base(p1.Shlib)] { + continue + } + haveShlib[filepath.Base(p1.Shlib)] = true + // TODO(rsc): The use of ModeInstall here is suspect, but if we only do ModeBuild, + // we'll end up building an overall library or executable that depends at runtime + // on other libraries that are out-of-date, which is clearly not good either. + // We call it ModeBuggyInstall to make clear that this is not right. + a.Deps = append(a.Deps, b.linkSharedAction(ModeBuggyInstall, ModeBuggyInstall, p1.Shlib, nil)) + } + } +} + +// addInstallHeaderAction adds an install header action to a, if needed. +// The action a should be an install action as generated by either +// b.CompileAction or b.LinkAction with mode=ModeInstall, +// and so a.Deps[0] is the corresponding build action. +func (b *Builder) addInstallHeaderAction(a *Action) { + // Install header for cgo in c-archive and c-shared modes. + p := a.Package + if p.UsesCgo() && (cfg.BuildBuildmode == "c-archive" || cfg.BuildBuildmode == "c-shared") { + hdrTarget := a.Target[:len(a.Target)-len(filepath.Ext(a.Target))] + ".h" + if cfg.BuildContext.Compiler == "gccgo" && cfg.BuildO == "" { + // For the header file, remove the "lib" + // added by go/build, so we generate pkg.h + // rather than libpkg.h. + dir, file := filepath.Split(hdrTarget) + file = strings.TrimPrefix(file, "lib") + hdrTarget = filepath.Join(dir, file) + } + ah := &Action{ + Mode: "install header", + Package: a.Package, + Deps: []*Action{a.Deps[0]}, + Actor: ActorFunc((*Builder).installHeader), + Objdir: a.Deps[0].Objdir, + Target: hdrTarget, + } + a.Deps = append(a.Deps, ah) + } +} + +// buildmodeShared takes the "go build" action a1 into the building of a shared library of a1.Deps. +// That is, the input a1 represents "go build pkgs" and the result represents "go build -buildmode=shared pkgs". +func (b *Builder) buildmodeShared(mode, depMode BuildMode, args []string, pkgs []*load.Package, a1 *Action) *Action { + name, err := libname(args, pkgs) + if err != nil { + base.Fatalf("%v", err) + } + return b.linkSharedAction(mode, depMode, name, a1) +} + +// linkSharedAction takes a grouping action a1 corresponding to a list of built packages +// and returns an action that links them together into a shared library with the name shlib. +// If a1 is nil, shlib should be an absolute path to an existing shared library, +// and then linkSharedAction reads that library to find out the package list. +func (b *Builder) linkSharedAction(mode, depMode BuildMode, shlib string, a1 *Action) *Action { + fullShlib := shlib + shlib = filepath.Base(shlib) + a := b.cacheAction("build-shlib "+shlib, nil, func() *Action { + if a1 == nil { + // TODO(rsc): Need to find some other place to store config, + // not in pkg directory. See golang.org/issue/22196. + pkgs := readpkglist(fullShlib) + a1 = &Action{ + Mode: "shlib packages", + } + for _, p := range pkgs { + a1.Deps = append(a1.Deps, b.CompileAction(mode, depMode, p)) + } + } + + // Fake package to hold ldflags. + // As usual shared libraries are a kludgy, abstraction-violating special case: + // we let them use the flags specified for the command-line arguments. + p := &load.Package{} + p.Internal.CmdlinePkg = true + p.Internal.Ldflags = load.BuildLdflags.For(p) + p.Internal.Gccgoflags = load.BuildGccgoflags.For(p) + + // Add implicit dependencies to pkgs list. + // Currently buildmode=shared forces external linking mode, and + // external linking mode forces an import of runtime/cgo (and + // math on arm). So if it was not passed on the command line and + // it is not present in another shared library, add it here. + // TODO(rsc): Maybe this should only happen if "runtime" is in the original package set. + // TODO(rsc): This should probably be changed to use load.LinkerDeps(p). + // TODO(rsc): We don't add standard library imports for gccgo + // because they are all always linked in anyhow. + // Maybe load.LinkerDeps should be used and updated. + a := &Action{ + Mode: "go build -buildmode=shared", + Package: p, + Objdir: b.NewObjdir(), + Actor: ActorFunc((*Builder).linkShared), + Deps: []*Action{a1}, + } + a.Target = filepath.Join(a.Objdir, shlib) + if cfg.BuildToolchainName != "gccgo" { + add := func(a1 *Action, pkg string, force bool) { + for _, a2 := range a1.Deps { + if a2.Package != nil && a2.Package.ImportPath == pkg { + return + } + } + var stk load.ImportStack + p := load.LoadPackageWithFlags(pkg, base.Cwd(), &stk, nil, 0) + if p.Error != nil { + base.Fatalf("load %s: %v", pkg, p.Error) + } + // Assume that if pkg (runtime/cgo or math) + // is already accounted for in a different shared library, + // then that shared library also contains runtime, + // so that anything we do will depend on that library, + // so we don't need to include pkg in our shared library. + if force || p.Shlib == "" || filepath.Base(p.Shlib) == pkg { + a1.Deps = append(a1.Deps, b.CompileAction(depMode, depMode, p)) + } + } + add(a1, "runtime/cgo", false) + if cfg.Goarch == "arm" { + add(a1, "math", false) + } + + // The linker step still needs all the usual linker deps. + // (For example, the linker always opens runtime.a.) + ldDeps, err := load.LinkerDeps(nil) + if err != nil { + base.Error(err) + } + for _, dep := range ldDeps { + add(a, dep, true) + } + } + b.addTransitiveLinkDeps(a, a1, shlib) + return a + }) + + // Install result. + if (mode == ModeInstall || mode == ModeBuggyInstall) && a.Actor != nil { + buildAction := a + + a = b.cacheAction("install-shlib "+shlib, nil, func() *Action { + // Determine the eventual install target. + // The install target is root/pkg/shlib, where root is the source root + // in which all the packages lie. + // TODO(rsc): Perhaps this cross-root check should apply to the full + // transitive package dependency list, not just the ones named + // on the command line? + pkgDir := a1.Deps[0].Package.Internal.Build.PkgTargetRoot + for _, a2 := range a1.Deps { + if dir := a2.Package.Internal.Build.PkgTargetRoot; dir != pkgDir { + base.Fatalf("installing shared library: cannot use packages %s and %s from different roots %s and %s", + a1.Deps[0].Package.ImportPath, + a2.Package.ImportPath, + pkgDir, + dir) + } + } + // TODO(rsc): Find out and explain here why gccgo is different. + if cfg.BuildToolchainName == "gccgo" { + pkgDir = filepath.Join(pkgDir, "shlibs") + } + target := filepath.Join(pkgDir, shlib) + + a := &Action{ + Mode: "go install -buildmode=shared", + Objdir: buildAction.Objdir, + Actor: ActorFunc(BuildInstallFunc), + Deps: []*Action{buildAction}, + Target: target, + } + for _, a2 := range buildAction.Deps[0].Deps { + p := a2.Package + pkgTargetRoot := p.Internal.Build.PkgTargetRoot + if pkgTargetRoot == "" { + continue + } + a.Deps = append(a.Deps, &Action{ + Mode: "shlibname", + Package: p, + Actor: ActorFunc((*Builder).installShlibname), + Target: filepath.Join(pkgTargetRoot, p.ImportPath+".shlibname"), + Deps: []*Action{a.Deps[0]}, + }) + } + return a + }) + } + + return a +} diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go new file mode 100644 index 0000000..408edb5 --- /dev/null +++ b/src/cmd/go/internal/work/build.go @@ -0,0 +1,961 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "context" + "errors" + "flag" + "fmt" + "go/build" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/load" + "cmd/go/internal/modload" + "cmd/go/internal/search" + "cmd/go/internal/trace" +) + +var CmdBuild = &base.Command{ + UsageLine: "go build [-o output] [build flags] [packages]", + Short: "compile packages and dependencies", + Long: ` +Build compiles the packages named by the import paths, +along with their dependencies, but it does not install the results. + +If the arguments to build are a list of .go files from a single directory, +build treats them as a list of source files specifying a single package. + +When compiling packages, build ignores files that end in '_test.go'. + +When compiling a single main package, build writes the resulting +executable to an output file named after the last non-major-version +component of the package import path. The '.exe' suffix is added +when writing a Windows executable. +So 'go build example/sam' writes 'sam' or 'sam.exe'. +'go build example.com/foo/v2' writes 'foo' or 'foo.exe', not 'v2.exe'. + +When compiling a package from a list of .go files, the executable +is named after the first source file. +'go build ed.go rx.go' writes 'ed' or 'ed.exe'. + +When compiling multiple packages or a single non-main package, +build compiles the packages but discards the resulting object, +serving only as a check that the packages can be built. + +The -o flag forces build to write the resulting executable or object +to the named output file or directory, instead of the default behavior described +in the last two paragraphs. If the named output is an existing directory or +ends with a slash or backslash, then any resulting executables +will be written to that directory. + +The build flags are shared by the build, clean, get, install, list, run, +and test commands: + + -C dir + Change to dir before running the command. + Any files named on the command line are interpreted after + changing directories. + If used, this flag must be the first one in the command line. + -a + force rebuilding of packages that are already up-to-date. + -n + print the commands but do not run them. + -p n + the number of programs, such as build commands or + test binaries, that can be run in parallel. + The default is GOMAXPROCS, normally the number of CPUs available. + -race + enable data race detection. + Supported only on linux/amd64, freebsd/amd64, darwin/amd64, darwin/arm64, windows/amd64, + linux/ppc64le and linux/arm64 (only for 48-bit VMA). + -msan + enable interoperation with memory sanitizer. + Supported only on linux/amd64, linux/arm64, linux/loong64, freebsd/amd64 + and only with Clang/LLVM as the host C compiler. + PIE build mode will be used on all platforms except linux/amd64. + -asan + enable interoperation with address sanitizer. + Supported only on linux/arm64, linux/amd64, linux/loong64. + Supported on linux/amd64 or linux/arm64 and only with GCC 7 and higher + or Clang/LLVM 9 and higher. + And supported on linux/loong64 only with Clang/LLVM 16 and higher. + -cover + enable code coverage instrumentation. + -covermode set,count,atomic + set the mode for coverage analysis. + The default is "set" unless -race is enabled, + in which case it is "atomic". + The values: + set: bool: does this statement run? + count: int: how many times does this statement run? + atomic: int: count, but correct in multithreaded tests; + significantly more expensive. + Sets -cover. + -coverpkg pattern1,pattern2,pattern3 + For a build that targets package 'main' (e.g. building a Go + executable), apply coverage analysis to each package matching + the patterns. The default is to apply coverage analysis to + packages in the main Go module. See 'go help packages' for a + description of package patterns. Sets -cover. + -v + print the names of packages as they are compiled. + -work + print the name of the temporary work directory and + do not delete it when exiting. + -x + print the commands. + -asmflags '[pattern=]arg list' + arguments to pass on each go tool asm invocation. + -buildmode mode + build mode to use. See 'go help buildmode' for more. + -buildvcs + Whether to stamp binaries with version control information + ("true", "false", or "auto"). By default ("auto"), version control + information is stamped into a binary if the main package, the main module + containing it, and the current directory are all in the same repository. + Use -buildvcs=false to always omit version control information, or + -buildvcs=true to error out if version control information is available but + cannot be included due to a missing tool or ambiguous directory structure. + -compiler name + name of compiler to use, as in runtime.Compiler (gccgo or gc). + -gccgoflags '[pattern=]arg list' + arguments to pass on each gccgo compiler/linker invocation. + -gcflags '[pattern=]arg list' + arguments to pass on each go tool compile invocation. + -installsuffix suffix + a suffix to use in the name of the package installation directory, + in order to keep output separate from default builds. + If using the -race flag, the install suffix is automatically set to race + or, if set explicitly, has _race appended to it. Likewise for the -msan + and -asan flags. Using a -buildmode option that requires non-default compile + flags has a similar effect. + -ldflags '[pattern=]arg list' + arguments to pass on each go tool link invocation. + -linkshared + build code that will be linked against shared libraries previously + created with -buildmode=shared. + -mod mode + module download mode to use: readonly, vendor, or mod. + By default, if a vendor directory is present and the go version in go.mod + is 1.14 or higher, the go command acts as if -mod=vendor were set. + Otherwise, the go command acts as if -mod=readonly were set. + See https://golang.org/ref/mod#build-commands for details. + -modcacherw + leave newly-created directories in the module cache read-write + instead of making them read-only. + -modfile file + in module aware mode, read (and possibly write) an alternate go.mod + file instead of the one in the module root directory. A file named + "go.mod" must still be present in order to determine the module root + directory, but it is not accessed. When -modfile is specified, an + alternate go.sum file is also used: its path is derived from the + -modfile flag by trimming the ".mod" extension and appending ".sum". + -overlay file + read a JSON config file that provides an overlay for build operations. + The file is a JSON struct with a single field, named 'Replace', that + maps each disk file path (a string) to its backing file path, so that + a build will run as if the disk file path exists with the contents + given by the backing file paths, or as if the disk file path does not + exist if its backing file path is empty. Support for the -overlay flag + has some limitations: importantly, cgo files included from outside the + include path must be in the same directory as the Go package they are + included from, and overlays will not appear when binaries and tests are + run through go run and go test respectively. + -pgo file + specify the file path of a profile for profile-guided optimization (PGO). + When the special name "auto" is specified, for each main package in the + build, the go command selects a file named "default.pgo" in the package's + directory if that file exists, and applies it to the (transitive) + dependencies of the main package (other packages are not affected). + Special name "off" turns off PGO. The default is "auto". + -pkgdir dir + install and load all packages from dir instead of the usual locations. + For example, when building with a non-standard configuration, + use -pkgdir to keep generated packages in a separate location. + -tags tag,list + a comma-separated list of additional build tags to consider satisfied + during the build. For more information about build tags, see + 'go help buildconstraint'. (Earlier versions of Go used a + space-separated list, and that form is deprecated but still recognized.) + -trimpath + remove all file system paths from the resulting executable. + Instead of absolute file system paths, the recorded file names + will begin either a module path@version (when using modules), + or a plain import path (when using the standard library, or GOPATH). + -toolexec 'cmd args' + a program to use to invoke toolchain programs like vet and asm. + For example, instead of running asm, the go command will run + 'cmd args /path/to/asm <arguments for asm>'. + The TOOLEXEC_IMPORTPATH environment variable will be set, + matching 'go list -f {{.ImportPath}}' for the package being built. + +The -asmflags, -gccgoflags, -gcflags, and -ldflags flags accept a +space-separated list of arguments to pass to an underlying tool +during the build. To embed spaces in an element in the list, surround +it with either single or double quotes. The argument list may be +preceded by a package pattern and an equal sign, which restricts +the use of that argument list to the building of packages matching +that pattern (see 'go help packages' for a description of package +patterns). Without a pattern, the argument list applies only to the +packages named on the command line. The flags may be repeated +with different patterns in order to specify different arguments for +different sets of packages. If a package matches patterns given in +multiple flags, the latest match on the command line wins. +For example, 'go build -gcflags=-S fmt' prints the disassembly +only for package fmt, while 'go build -gcflags=all=-S fmt' +prints the disassembly for fmt and all its dependencies. + +For more about specifying packages, see 'go help packages'. +For more about where packages and binaries are installed, +run 'go help gopath'. +For more about calling between Go and C/C++, run 'go help c'. + +Note: Build adheres to certain conventions such as those described +by 'go help gopath'. Not all projects can follow these conventions, +however. Installations that have their own conventions or that use +a separate software build system may choose to use lower-level +invocations such as 'go tool compile' and 'go tool link' to avoid +some of the overheads and design decisions of the build tool. + +See also: go install, go get, go clean. + `, +} + +const concurrentGCBackendCompilationEnabledByDefault = true + +func init() { + // break init cycle + CmdBuild.Run = runBuild + CmdInstall.Run = runInstall + + CmdBuild.Flag.StringVar(&cfg.BuildO, "o", "", "output file or directory") + + AddBuildFlags(CmdBuild, DefaultBuildFlags) + AddBuildFlags(CmdInstall, DefaultBuildFlags) + if cfg.Experiment != nil && cfg.Experiment.CoverageRedesign { + AddCoverFlags(CmdBuild, nil) + AddCoverFlags(CmdInstall, nil) + } +} + +// Note that flags consulted by other parts of the code +// (for example, buildV) are in cmd/go/internal/cfg. + +var ( + forcedAsmflags []string // internally-forced flags for cmd/asm + forcedGcflags []string // internally-forced flags for cmd/compile + forcedLdflags []string // internally-forced flags for cmd/link + forcedGccgoflags []string // internally-forced flags for gccgo +) + +var BuildToolchain toolchain = noToolchain{} +var ldBuildmode string + +// buildCompiler implements flag.Var. +// It implements Set by updating both +// BuildToolchain and buildContext.Compiler. +type buildCompiler struct{} + +func (c buildCompiler) Set(value string) error { + switch value { + case "gc": + BuildToolchain = gcToolchain{} + case "gccgo": + BuildToolchain = gccgoToolchain{} + default: + return fmt.Errorf("unknown compiler %q", value) + } + cfg.BuildToolchainName = value + cfg.BuildToolchainCompiler = BuildToolchain.compiler + cfg.BuildToolchainLinker = BuildToolchain.linker + cfg.BuildContext.Compiler = value + return nil +} + +func (c buildCompiler) String() string { + return cfg.BuildContext.Compiler +} + +func init() { + switch build.Default.Compiler { + case "gc", "gccgo": + buildCompiler{}.Set(build.Default.Compiler) + } +} + +type BuildFlagMask int + +const ( + DefaultBuildFlags BuildFlagMask = 0 + OmitModFlag BuildFlagMask = 1 << iota + OmitModCommonFlags + OmitVFlag +) + +// AddBuildFlags adds the flags common to the build, clean, get, +// install, list, run, and test commands. +func AddBuildFlags(cmd *base.Command, mask BuildFlagMask) { + base.AddBuildFlagsNX(&cmd.Flag) + base.AddChdirFlag(&cmd.Flag) + cmd.Flag.BoolVar(&cfg.BuildA, "a", false, "") + cmd.Flag.IntVar(&cfg.BuildP, "p", cfg.BuildP, "") + if mask&OmitVFlag == 0 { + cmd.Flag.BoolVar(&cfg.BuildV, "v", false, "") + } + + cmd.Flag.Var(&load.BuildAsmflags, "asmflags", "") + cmd.Flag.Var(buildCompiler{}, "compiler", "") + cmd.Flag.StringVar(&cfg.BuildBuildmode, "buildmode", "default", "") + cmd.Flag.Var(&load.BuildGcflags, "gcflags", "") + cmd.Flag.Var(&load.BuildGccgoflags, "gccgoflags", "") + if mask&OmitModFlag == 0 { + base.AddModFlag(&cmd.Flag) + } + if mask&OmitModCommonFlags == 0 { + base.AddModCommonFlags(&cmd.Flag) + } else { + // Add the overlay flag even when we don't add the rest of the mod common flags. + // This only affects 'go get' in GOPATH mode, but add the flag anyway for + // consistency. + cmd.Flag.StringVar(&fsys.OverlayFile, "overlay", "", "") + } + cmd.Flag.StringVar(&cfg.BuildContext.InstallSuffix, "installsuffix", "", "") + cmd.Flag.Var(&load.BuildLdflags, "ldflags", "") + cmd.Flag.BoolVar(&cfg.BuildLinkshared, "linkshared", false, "") + cmd.Flag.StringVar(&cfg.BuildPGO, "pgo", "auto", "") + cmd.Flag.StringVar(&cfg.BuildPkgdir, "pkgdir", "", "") + cmd.Flag.BoolVar(&cfg.BuildRace, "race", false, "") + cmd.Flag.BoolVar(&cfg.BuildMSan, "msan", false, "") + cmd.Flag.BoolVar(&cfg.BuildASan, "asan", false, "") + cmd.Flag.Var((*tagsFlag)(&cfg.BuildContext.BuildTags), "tags", "") + cmd.Flag.Var((*base.StringsFlag)(&cfg.BuildToolexec), "toolexec", "") + cmd.Flag.BoolVar(&cfg.BuildTrimpath, "trimpath", false, "") + cmd.Flag.BoolVar(&cfg.BuildWork, "work", false, "") + cmd.Flag.Var((*buildvcsFlag)(&cfg.BuildBuildvcs), "buildvcs", "") + + // Undocumented, unstable debugging flags. + cmd.Flag.StringVar(&cfg.DebugActiongraph, "debug-actiongraph", "", "") + cmd.Flag.StringVar(&cfg.DebugTrace, "debug-trace", "", "") + cmd.Flag.StringVar(&cfg.DebugRuntimeTrace, "debug-runtime-trace", "", "") +} + +// AddCoverFlags adds coverage-related flags to "cmd". If the +// CoverageRedesign experiment is enabled, we add -cover{mode,pkg} to +// the build command and only -coverprofile to the test command. If +// the CoverageRedesign experiment is disabled, -cover* flags are +// added only to the test command. +func AddCoverFlags(cmd *base.Command, coverProfileFlag *string) { + addCover := false + if cfg.Experiment != nil && cfg.Experiment.CoverageRedesign { + // New coverage enabled: both build and test commands get + // coverage flags. + addCover = true + } else { + // New coverage disabled: only test command gets cover flags. + addCover = coverProfileFlag != nil + } + if addCover { + cmd.Flag.BoolVar(&cfg.BuildCover, "cover", false, "") + cmd.Flag.Var(coverFlag{(*coverModeFlag)(&cfg.BuildCoverMode)}, "covermode", "") + cmd.Flag.Var(coverFlag{commaListFlag{&cfg.BuildCoverPkg}}, "coverpkg", "") + } + if coverProfileFlag != nil { + cmd.Flag.Var(coverFlag{V: stringFlag{coverProfileFlag}}, "coverprofile", "") + } +} + +// tagsFlag is the implementation of the -tags flag. +type tagsFlag []string + +func (v *tagsFlag) Set(s string) error { + // For compatibility with Go 1.12 and earlier, allow "-tags='a b c'" or even just "-tags='a'". + if strings.Contains(s, " ") || strings.Contains(s, "'") { + return (*base.StringsFlag)(v).Set(s) + } + + // Split on commas, ignore empty strings. + *v = []string{} + for _, s := range strings.Split(s, ",") { + if s != "" { + *v = append(*v, s) + } + } + return nil +} + +func (v *tagsFlag) String() string { + return "<TagsFlag>" +} + +// buildvcsFlag is the implementation of the -buildvcs flag. +type buildvcsFlag string + +func (f *buildvcsFlag) IsBoolFlag() bool { return true } // allow -buildvcs (without arguments) + +func (f *buildvcsFlag) Set(s string) error { + // https://go.dev/issue/51748: allow "-buildvcs=auto", + // in addition to the usual "true" and "false". + if s == "" || s == "auto" { + *f = "auto" + return nil + } + + b, err := strconv.ParseBool(s) + if err != nil { + return errors.New("value is neither 'auto' nor a valid bool") + } + *f = (buildvcsFlag)(strconv.FormatBool(b)) // convert to canonical "true" or "false" + return nil +} + +func (f *buildvcsFlag) String() string { return string(*f) } + +// fileExtSplit expects a filename and returns the name +// and ext (without the dot). If the file has no +// extension, ext will be empty. +func fileExtSplit(file string) (name, ext string) { + dotExt := filepath.Ext(file) + name = file[:len(file)-len(dotExt)] + if dotExt != "" { + ext = dotExt[1:] + } + return +} + +func pkgsMain(pkgs []*load.Package) (res []*load.Package) { + for _, p := range pkgs { + if p.Name == "main" { + res = append(res, p) + } + } + return res +} + +func pkgsNotMain(pkgs []*load.Package) (res []*load.Package) { + for _, p := range pkgs { + if p.Name != "main" { + res = append(res, p) + } + } + return res +} + +func oneMainPkg(pkgs []*load.Package) []*load.Package { + if len(pkgs) != 1 || pkgs[0].Name != "main" { + base.Fatalf("-buildmode=%s requires exactly one main package", cfg.BuildBuildmode) + } + return pkgs +} + +var pkgsFilter = func(pkgs []*load.Package) []*load.Package { return pkgs } + +func runBuild(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + BuildInit() + b := NewBuilder("") + defer func() { + if err := b.Close(); err != nil { + base.Fatal(err) + } + }() + + pkgs := load.PackagesAndErrors(ctx, load.PackageOpts{AutoVCS: true}, args) + load.CheckPackageErrors(pkgs) + + explicitO := len(cfg.BuildO) > 0 + + if len(pkgs) == 1 && pkgs[0].Name == "main" && cfg.BuildO == "" { + cfg.BuildO = pkgs[0].DefaultExecName() + cfg.BuildO += cfg.ExeSuffix + } + + // sanity check some often mis-used options + switch cfg.BuildContext.Compiler { + case "gccgo": + if load.BuildGcflags.Present() { + fmt.Println("go build: when using gccgo toolchain, please pass compiler flags using -gccgoflags, not -gcflags") + } + if load.BuildLdflags.Present() { + fmt.Println("go build: when using gccgo toolchain, please pass linker flags using -gccgoflags, not -ldflags") + } + case "gc": + if load.BuildGccgoflags.Present() { + fmt.Println("go build: when using gc toolchain, please pass compile flags using -gcflags, and linker flags using -ldflags") + } + } + + depMode := ModeBuild + + pkgs = omitTestOnly(pkgsFilter(pkgs)) + + // Special case -o /dev/null by not writing at all. + if base.IsNull(cfg.BuildO) { + cfg.BuildO = "" + } + + if cfg.Experiment.CoverageRedesign && cfg.BuildCover { + load.PrepareForCoverageBuild(pkgs) + } + + if cfg.BuildO != "" { + // If the -o name exists and is a directory or + // ends with a slash or backslash, then + // write all main packages to that directory. + // Otherwise require only a single package be built. + if fi, err := os.Stat(cfg.BuildO); (err == nil && fi.IsDir()) || + strings.HasSuffix(cfg.BuildO, "/") || + strings.HasSuffix(cfg.BuildO, string(os.PathSeparator)) { + if !explicitO { + base.Fatalf("go: build output %q already exists and is a directory", cfg.BuildO) + } + a := &Action{Mode: "go build"} + for _, p := range pkgs { + if p.Name != "main" { + continue + } + + p.Target = filepath.Join(cfg.BuildO, p.DefaultExecName()) + p.Target += cfg.ExeSuffix + p.Stale = true + p.StaleReason = "build -o flag in use" + a.Deps = append(a.Deps, b.AutoAction(ModeInstall, depMode, p)) + } + if len(a.Deps) == 0 { + base.Fatalf("go: no main packages to build") + } + b.Do(ctx, a) + return + } + if len(pkgs) > 1 { + base.Fatalf("go: cannot write multiple packages to non-directory %s", cfg.BuildO) + } else if len(pkgs) == 0 { + base.Fatalf("no packages to build") + } + p := pkgs[0] + p.Target = cfg.BuildO + p.Stale = true // must build - not up to date + p.StaleReason = "build -o flag in use" + a := b.AutoAction(ModeInstall, depMode, p) + b.Do(ctx, a) + return + } + + a := &Action{Mode: "go build"} + for _, p := range pkgs { + a.Deps = append(a.Deps, b.AutoAction(ModeBuild, depMode, p)) + } + if cfg.BuildBuildmode == "shared" { + a = b.buildmodeShared(ModeBuild, depMode, args, pkgs, a) + } + b.Do(ctx, a) +} + +var CmdInstall = &base.Command{ + UsageLine: "go install [build flags] [packages]", + Short: "compile and install packages and dependencies", + Long: ` +Install compiles and installs the packages named by the import paths. + +Executables are installed in the directory named by the GOBIN environment +variable, which defaults to $GOPATH/bin or $HOME/go/bin if the GOPATH +environment variable is not set. Executables in $GOROOT +are installed in $GOROOT/bin or $GOTOOLDIR instead of $GOBIN. + +If the arguments have version suffixes (like @latest or @v1.0.0), "go install" +builds packages in module-aware mode, ignoring the go.mod file in the current +directory or any parent directory, if there is one. This is useful for +installing executables without affecting the dependencies of the main module. +To eliminate ambiguity about which module versions are used in the build, the +arguments must satisfy the following constraints: + +- Arguments must be package paths or package patterns (with "..." wildcards). +They must not be standard packages (like fmt), meta-patterns (std, cmd, +all), or relative or absolute file paths. + +- All arguments must have the same version suffix. Different queries are not +allowed, even if they refer to the same version. + +- All arguments must refer to packages in the same module at the same version. + +- Package path arguments must refer to main packages. Pattern arguments +will only match main packages. + +- No module is considered the "main" module. If the module containing +packages named on the command line has a go.mod file, it must not contain +directives (replace and exclude) that would cause it to be interpreted +differently than if it were the main module. The module must not require +a higher version of itself. + +- Vendor directories are not used in any module. (Vendor directories are not +included in the module zip files downloaded by 'go install'.) + +If the arguments don't have version suffixes, "go install" may run in +module-aware mode or GOPATH mode, depending on the GO111MODULE environment +variable and the presence of a go.mod file. See 'go help modules' for details. +If module-aware mode is enabled, "go install" runs in the context of the main +module. + +When module-aware mode is disabled, non-main packages are installed in the +directory $GOPATH/pkg/$GOOS_$GOARCH. When module-aware mode is enabled, +non-main packages are built and cached but not installed. + +Before Go 1.20, the standard library was installed to +$GOROOT/pkg/$GOOS_$GOARCH. +Starting in Go 1.20, the standard library is built and cached but not installed. +Setting GODEBUG=installgoroot=all restores the use of +$GOROOT/pkg/$GOOS_$GOARCH. + +For more about build flags, see 'go help build'. + +For more about specifying packages, see 'go help packages'. + +See also: go build, go get, go clean. + `, +} + +// libname returns the filename to use for the shared library when using +// -buildmode=shared. The rules we use are: +// Use arguments for special 'meta' packages: +// +// std --> libstd.so +// std cmd --> libstd,cmd.so +// +// A single non-meta argument with trailing "/..." is special cased: +// +// foo/... --> libfoo.so +// (A relative path like "./..." expands the "." first) +// +// Use import paths for other cases, changing '/' to '-': +// +// somelib --> libsubdir-somelib.so +// ./ or ../ --> libsubdir-somelib.so +// gopkg.in/tomb.v2 -> libgopkg.in-tomb.v2.so +// a/... b/... ---> liba/c,b/d.so - all matching import paths +// +// Name parts are joined with ','. +func libname(args []string, pkgs []*load.Package) (string, error) { + var libname string + appendName := func(arg string) { + if libname == "" { + libname = arg + } else { + libname += "," + arg + } + } + var haveNonMeta bool + for _, arg := range args { + if search.IsMetaPackage(arg) { + appendName(arg) + } else { + haveNonMeta = true + } + } + if len(libname) == 0 { // non-meta packages only. use import paths + if len(args) == 1 && strings.HasSuffix(args[0], "/...") { + // Special case of "foo/..." as mentioned above. + arg := strings.TrimSuffix(args[0], "/...") + if build.IsLocalImport(arg) { + cwd, _ := os.Getwd() + bp, _ := cfg.BuildContext.ImportDir(filepath.Join(cwd, arg), build.FindOnly) + if bp.ImportPath != "" && bp.ImportPath != "." { + arg = bp.ImportPath + } + } + appendName(strings.ReplaceAll(arg, "/", "-")) + } else { + for _, pkg := range pkgs { + appendName(strings.ReplaceAll(pkg.ImportPath, "/", "-")) + } + } + } else if haveNonMeta { // have both meta package and a non-meta one + return "", errors.New("mixing of meta and non-meta packages is not allowed") + } + // TODO(mwhudson): Needs to change for platforms that use different naming + // conventions... + return "lib" + libname + ".so", nil +} + +func runInstall(ctx context.Context, cmd *base.Command, args []string) { + for _, arg := range args { + if strings.Contains(arg, "@") && !build.IsLocalImport(arg) && !filepath.IsAbs(arg) { + installOutsideModule(ctx, args) + return + } + } + + modload.InitWorkfile() + BuildInit() + pkgs := load.PackagesAndErrors(ctx, load.PackageOpts{AutoVCS: true}, args) + if cfg.ModulesEnabled && !modload.HasModRoot() { + haveErrors := false + allMissingErrors := true + for _, pkg := range pkgs { + if pkg.Error == nil { + continue + } + haveErrors = true + if missingErr := (*modload.ImportMissingError)(nil); !errors.As(pkg.Error, &missingErr) { + allMissingErrors = false + break + } + } + if haveErrors && allMissingErrors { + latestArgs := make([]string, len(args)) + for i := range args { + latestArgs[i] = args[i] + "@latest" + } + hint := strings.Join(latestArgs, " ") + base.Fatalf("go: 'go install' requires a version when current directory is not in a module\n\tTry 'go install %s' to install the latest version", hint) + } + } + load.CheckPackageErrors(pkgs) + + if cfg.Experiment.CoverageRedesign && cfg.BuildCover { + load.PrepareForCoverageBuild(pkgs) + } + + InstallPackages(ctx, args, pkgs) +} + +// omitTestOnly returns pkgs with test-only packages removed. +func omitTestOnly(pkgs []*load.Package) []*load.Package { + var list []*load.Package + for _, p := range pkgs { + if len(p.GoFiles)+len(p.CgoFiles) == 0 && !p.Internal.CmdlinePkgLiteral { + // Package has no source files, + // perhaps due to build tags or perhaps due to only having *_test.go files. + // Also, it is only being processed as the result of a wildcard match + // like ./..., not because it was listed as a literal path on the command line. + // Ignore it. + continue + } + list = append(list, p) + } + return list +} + +func InstallPackages(ctx context.Context, patterns []string, pkgs []*load.Package) { + ctx, span := trace.StartSpan(ctx, "InstallPackages "+strings.Join(patterns, " ")) + defer span.Done() + + if cfg.GOBIN != "" && !filepath.IsAbs(cfg.GOBIN) { + base.Fatalf("cannot install, GOBIN must be an absolute path") + } + + pkgs = omitTestOnly(pkgsFilter(pkgs)) + for _, p := range pkgs { + if p.Target == "" { + switch { + case p.Name != "main" && p.Internal.Local && p.ConflictDir == "": + // Non-executables outside GOPATH need not have a target: + // we can use the cache to hold the built package archive for use in future builds. + // The ones inside GOPATH should have a target (in GOPATH/pkg) + // or else something is wrong and worth reporting (like a ConflictDir). + case p.Name != "main" && p.Module != nil: + // Non-executables have no target (except the cache) when building with modules. + case p.Name != "main" && p.Standard && p.Internal.Build.PkgObj == "": + // Most packages in std do not need an installed .a, because they can be + // rebuilt and used directly from the build cache. + // A few targets (notably those using cgo) still do need to be installed + // in case the user's environment lacks a C compiler. + case p.Internal.GobinSubdir: + base.Errorf("go: cannot install cross-compiled binaries when GOBIN is set") + case p.Internal.CmdlineFiles: + base.Errorf("go: no install location for .go files listed on command line (GOBIN not set)") + case p.ConflictDir != "": + base.Errorf("go: no install location for %s: hidden by %s", p.Dir, p.ConflictDir) + default: + base.Errorf("go: no install location for directory %s outside GOPATH\n"+ + "\tFor more details see: 'go help gopath'", p.Dir) + } + } + } + base.ExitIfErrors() + + b := NewBuilder("") + defer func() { + if err := b.Close(); err != nil { + base.Fatal(err) + } + }() + + depMode := ModeBuild + a := &Action{Mode: "go install"} + var tools []*Action + for _, p := range pkgs { + // If p is a tool, delay the installation until the end of the build. + // This avoids installing assemblers/compilers that are being executed + // by other steps in the build. + a1 := b.AutoAction(ModeInstall, depMode, p) + if load.InstallTargetDir(p) == load.ToTool { + a.Deps = append(a.Deps, a1.Deps...) + a1.Deps = append(a1.Deps, a) + tools = append(tools, a1) + continue + } + a.Deps = append(a.Deps, a1) + } + if len(tools) > 0 { + a = &Action{ + Mode: "go install (tools)", + Deps: tools, + } + } + + if cfg.BuildBuildmode == "shared" { + // Note: If buildmode=shared then only non-main packages + // are present in the pkgs list, so all the special case code about + // tools above did not apply, and a is just a simple Action + // with a list of Deps, one per package named in pkgs, + // the same as in runBuild. + a = b.buildmodeShared(ModeInstall, ModeInstall, patterns, pkgs, a) + } + + b.Do(ctx, a) + base.ExitIfErrors() + + // Success. If this command is 'go install' with no arguments + // and the current directory (the implicit argument) is a command, + // remove any leftover command binary from a previous 'go build'. + // The binary is installed; it's not needed here anymore. + // And worse it might be a stale copy, which you don't want to find + // instead of the installed one if $PATH contains dot. + // One way to view this behavior is that it is as if 'go install' first + // runs 'go build' and the moves the generated file to the install dir. + // See issue 9645. + if len(patterns) == 0 && len(pkgs) == 1 && pkgs[0].Name == "main" { + // Compute file 'go build' would have created. + // If it exists and is an executable file, remove it. + targ := pkgs[0].DefaultExecName() + targ += cfg.ExeSuffix + if filepath.Join(pkgs[0].Dir, targ) != pkgs[0].Target { // maybe $GOBIN is the current directory + fi, err := os.Stat(targ) + if err == nil { + m := fi.Mode() + if m.IsRegular() { + if m&0111 != 0 || cfg.Goos == "windows" { // windows never sets executable bit + os.Remove(targ) + } + } + } + } + } +} + +// installOutsideModule implements 'go install pkg@version'. It builds and +// installs one or more main packages in module mode while ignoring any go.mod +// in the current directory or parent directories. +// +// See golang.org/issue/40276 for details and rationale. +func installOutsideModule(ctx context.Context, args []string) { + modload.ForceUseModules = true + modload.RootMode = modload.NoRoot + modload.AllowMissingModuleImports() + modload.Init() + BuildInit() + + // Load packages. Ignore non-main packages. + // Print a warning if an argument contains "..." and matches no main packages. + // PackagesAndErrors already prints warnings for patterns that don't match any + // packages, so be careful not to double print. + // TODO(golang.org/issue/40276): don't report errors loading non-main packages + // matched by a pattern. + pkgOpts := load.PackageOpts{MainOnly: true} + pkgs, err := load.PackagesAndErrorsOutsideModule(ctx, pkgOpts, args) + if err != nil { + base.Fatal(err) + } + load.CheckPackageErrors(pkgs) + patterns := make([]string, len(args)) + for i, arg := range args { + patterns[i] = arg[:strings.Index(arg, "@")] + } + + // Build and install the packages. + InstallPackages(ctx, patterns, pkgs) +} + +// ExecCmd is the command to use to run user binaries. +// Normally it is empty, meaning run the binaries directly. +// If cross-compiling and running on a remote system or +// simulator, it is typically go_GOOS_GOARCH_exec, with +// the target GOOS and GOARCH substituted. +// The -exec flag overrides these defaults. +var ExecCmd []string + +// FindExecCmd derives the value of ExecCmd to use. +// It returns that value and leaves ExecCmd set for direct use. +func FindExecCmd() []string { + if ExecCmd != nil { + return ExecCmd + } + ExecCmd = []string{} // avoid work the second time + if cfg.Goos == runtime.GOOS && cfg.Goarch == runtime.GOARCH { + return ExecCmd + } + path, err := cfg.LookPath(fmt.Sprintf("go_%s_%s_exec", cfg.Goos, cfg.Goarch)) + if err == nil { + ExecCmd = []string{path} + } + return ExecCmd +} + +// A coverFlag is a flag.Value that also implies -cover. +type coverFlag struct{ V flag.Value } + +func (f coverFlag) String() string { return f.V.String() } + +func (f coverFlag) Set(value string) error { + if err := f.V.Set(value); err != nil { + return err + } + cfg.BuildCover = true + return nil +} + +type coverModeFlag string + +func (f *coverModeFlag) String() string { return string(*f) } +func (f *coverModeFlag) Set(value string) error { + switch value { + case "", "set", "count", "atomic": + *f = coverModeFlag(value) + cfg.BuildCoverMode = value + return nil + default: + return errors.New(`valid modes are "set", "count", or "atomic"`) + } +} + +// A commaListFlag is a flag.Value representing a comma-separated list. +type commaListFlag struct{ Vals *[]string } + +func (f commaListFlag) String() string { return strings.Join(*f.Vals, ",") } + +func (f commaListFlag) Set(value string) error { + if value == "" { + *f.Vals = nil + } else { + *f.Vals = strings.Split(value, ",") + } + return nil +} + +// A stringFlag is a flag.Value representing a single string. +type stringFlag struct{ val *string } + +func (f stringFlag) String() string { return *f.val } +func (f stringFlag) Set(value string) error { + *f.val = value + return nil +} diff --git a/src/cmd/go/internal/work/build_test.go b/src/cmd/go/internal/work/build_test.go new file mode 100644 index 0000000..f3059f2 --- /dev/null +++ b/src/cmd/go/internal/work/build_test.go @@ -0,0 +1,281 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "fmt" + "internal/testenv" + "io/fs" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/load" +) + +func TestRemoveDevNull(t *testing.T) { + fi, err := os.Lstat(os.DevNull) + if err != nil { + t.Skip(err) + } + if fi.Mode().IsRegular() { + t.Errorf("Lstat(%s).Mode().IsRegular() = true; expected false", os.DevNull) + } + mayberemovefile(os.DevNull) + _, err = os.Lstat(os.DevNull) + if err != nil { + t.Errorf("mayberemovefile(%s) did remove it; oops", os.DevNull) + } +} + +func TestSplitPkgConfigOutput(t *testing.T) { + for _, test := range []struct { + in []byte + want []string + }{ + {[]byte(`-r:foo -L/usr/white\ space/lib -lfoo\ bar -lbar\ baz`), []string{"-r:foo", "-L/usr/white space/lib", "-lfoo bar", "-lbar baz"}}, + {[]byte(`-lextra\ fun\ arg\\`), []string{`-lextra fun arg\`}}, + {[]byte("\textra whitespace\r\n"), []string{"extra", "whitespace\r"}}, + {[]byte(" \r\n "), []string{"\r"}}, + {[]byte(`"-r:foo" "-L/usr/white space/lib" "-lfoo bar" "-lbar baz"`), []string{"-r:foo", "-L/usr/white space/lib", "-lfoo bar", "-lbar baz"}}, + {[]byte(`"-lextra fun arg\\"`), []string{`-lextra fun arg\`}}, + {[]byte(`" \r\n\ "`), []string{` \r\n\ `}}, + {[]byte(`""`), []string{""}}, + {[]byte(``), nil}, + {[]byte(`"\\"`), []string{`\`}}, + {[]byte(`"\x"`), []string{`\x`}}, + {[]byte(`"\\x"`), []string{`\x`}}, + {[]byte(`'\\'`), []string{`\\`}}, + {[]byte(`'\x'`), []string{`\x`}}, + {[]byte(`"\\x"`), []string{`\x`}}, + {[]byte("\\\n"), nil}, + {[]byte(`-fPIC -I/test/include/foo -DQUOTED='"/test/share/doc"'`), []string{"-fPIC", "-I/test/include/foo", `-DQUOTED="/test/share/doc"`}}, + {[]byte(`-fPIC -I/test/include/foo -DQUOTED="/test/share/doc"`), []string{"-fPIC", "-I/test/include/foo", "-DQUOTED=/test/share/doc"}}, + {[]byte(`-fPIC -I/test/include/foo -DQUOTED=\"/test/share/doc\"`), []string{"-fPIC", "-I/test/include/foo", `-DQUOTED="/test/share/doc"`}}, + {[]byte(`-fPIC -I/test/include/foo -DQUOTED='/test/share/doc'`), []string{"-fPIC", "-I/test/include/foo", "-DQUOTED=/test/share/doc"}}, + {[]byte(`-DQUOTED='/te\st/share/d\oc'`), []string{`-DQUOTED=/te\st/share/d\oc`}}, + {[]byte(`-Dhello=10 -Dworld=+32 -DDEFINED_FROM_PKG_CONFIG=hello\ world`), []string{"-Dhello=10", "-Dworld=+32", "-DDEFINED_FROM_PKG_CONFIG=hello world"}}, + {[]byte(`"broken\"" \\\a "a"`), []string{"broken\"", "\\a", "a"}}, + } { + got, err := splitPkgConfigOutput(test.in) + if err != nil { + t.Errorf("splitPkgConfigOutput on %#q failed with error %v", test.in, err) + continue + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("splitPkgConfigOutput(%#q) = %#q; want %#q", test.in, got, test.want) + } + } + + for _, test := range []struct { + in []byte + want []string + }{ + // broken quotation + {[]byte(`" \r\n `), nil}, + {[]byte(`"-r:foo" "-L/usr/white space/lib "-lfoo bar" "-lbar baz"`), nil}, + {[]byte(`"-lextra fun arg\\`), nil}, + // broken char escaping + {[]byte(`broken flag\`), nil}, + {[]byte(`extra broken flag \`), nil}, + {[]byte(`\`), nil}, + {[]byte(`"broken\"" "extra" \`), nil}, + } { + got, err := splitPkgConfigOutput(test.in) + if err == nil { + t.Errorf("splitPkgConfigOutput(%v) = %v; haven't failed with error as expected.", test.in, got) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("splitPkgConfigOutput(%v) = %v; want %v", test.in, got, test.want) + } + } + +} + +func TestSharedLibName(t *testing.T) { + // TODO(avdva) - make these values platform-specific + prefix := "lib" + suffix := ".so" + testData := []struct { + args []string + pkgs []*load.Package + expected string + expectErr bool + rootedAt string + }{ + { + args: []string{"std"}, + pkgs: []*load.Package{}, + expected: "std", + }, + { + args: []string{"std", "cmd"}, + pkgs: []*load.Package{}, + expected: "std,cmd", + }, + { + args: []string{}, + pkgs: []*load.Package{pkgImportPath("gopkg.in/somelib")}, + expected: "gopkg.in-somelib", + }, + { + args: []string{"./..."}, + pkgs: []*load.Package{pkgImportPath("somelib")}, + expected: "somelib", + rootedAt: "somelib", + }, + { + args: []string{"../somelib", "../somelib"}, + pkgs: []*load.Package{pkgImportPath("somelib")}, + expected: "somelib", + }, + { + args: []string{"../lib1", "../lib2"}, + pkgs: []*load.Package{pkgImportPath("gopkg.in/lib1"), pkgImportPath("gopkg.in/lib2")}, + expected: "gopkg.in-lib1,gopkg.in-lib2", + }, + { + args: []string{"./..."}, + pkgs: []*load.Package{ + pkgImportPath("gopkg.in/dir/lib1"), + pkgImportPath("gopkg.in/lib2"), + pkgImportPath("gopkg.in/lib3"), + }, + expected: "gopkg.in", + rootedAt: "gopkg.in", + }, + { + args: []string{"std", "../lib2"}, + pkgs: []*load.Package{}, + expectErr: true, + }, + { + args: []string{"all", "./"}, + pkgs: []*load.Package{}, + expectErr: true, + }, + { + args: []string{"cmd", "fmt"}, + pkgs: []*load.Package{}, + expectErr: true, + }, + } + for _, data := range testData { + func() { + if data.rootedAt != "" { + tmpGopath, err := os.MkdirTemp("", "gopath") + if err != nil { + t.Fatal(err) + } + cwd := base.Cwd() + oldGopath := cfg.BuildContext.GOPATH + defer func() { + cfg.BuildContext.GOPATH = oldGopath + os.Chdir(cwd) + err := os.RemoveAll(tmpGopath) + if err != nil { + t.Error(err) + } + }() + root := filepath.Join(tmpGopath, "src", data.rootedAt) + err = os.MkdirAll(root, 0755) + if err != nil { + t.Fatal(err) + } + cfg.BuildContext.GOPATH = tmpGopath + os.Chdir(root) + } + computed, err := libname(data.args, data.pkgs) + if err != nil { + if !data.expectErr { + t.Errorf("libname returned an error %q, expected a name", err.Error()) + } + } else if data.expectErr { + t.Errorf("libname returned %q, expected an error", computed) + } else { + expected := prefix + data.expected + suffix + if expected != computed { + t.Errorf("libname returned %q, expected %q", computed, expected) + } + } + }() + } +} + +func pkgImportPath(pkgpath string) *load.Package { + return &load.Package{ + PackagePublic: load.PackagePublic{ + ImportPath: pkgpath, + }, + } +} + +// When installing packages, the installed package directory should +// respect the SetGID bit and group name of the destination +// directory. +// See https://golang.org/issue/18878. +func TestRespectSetgidDir(t *testing.T) { + // Check that `cp` is called instead of `mv` by looking at the output + // of `(*Shell).ShowCmd` afterwards as a sanity check. + cfg.BuildX = true + var cmdBuf strings.Builder + sh := NewShell("", func(a ...any) (int, error) { + return cmdBuf.WriteString(fmt.Sprint(a...)) + }) + + setgiddir, err := os.MkdirTemp("", "SetGroupID") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(setgiddir) + + // BSD mkdir(2) inherits the parent directory group, and other platforms + // can inherit the parent directory group via setgid. The test setup (chmod + // setgid) will fail if the process does not have the group permission to + // the new temporary directory. + err = os.Chown(setgiddir, os.Getuid(), os.Getgid()) + if err != nil { + if testenv.SyscallIsNotSupported(err) { + t.Skip("skipping: chown is not supported on " + runtime.GOOS) + } + t.Fatal(err) + } + + // Change setgiddir's permissions to include the SetGID bit. + if err := os.Chmod(setgiddir, 0755|fs.ModeSetgid); err != nil { + if testenv.SyscallIsNotSupported(err) { + t.Skip("skipping: chmod is not supported on " + runtime.GOOS) + } + t.Fatal(err) + } + if fi, err := os.Stat(setgiddir); err != nil { + t.Fatal(err) + } else if fi.Mode()&fs.ModeSetgid == 0 { + t.Skip("skipping: Chmod ignored ModeSetgid on " + runtime.GOOS) + } + + pkgfile, err := os.CreateTemp("", "pkgfile") + if err != nil { + t.Fatalf("os.CreateTemp(\"\", \"pkgfile\"): %v", err) + } + defer os.Remove(pkgfile.Name()) + defer pkgfile.Close() + + dirGIDFile := filepath.Join(setgiddir, "setgid") + if err := sh.moveOrCopyFile(dirGIDFile, pkgfile.Name(), 0666, true); err != nil { + t.Fatalf("moveOrCopyFile: %v", err) + } + + got := strings.TrimSpace(cmdBuf.String()) + want := sh.fmtCmd("", "cp %s %s", pkgfile.Name(), dirGIDFile) + if got != want { + t.Fatalf("moveOrCopyFile(%q, %q): want %q, got %q", dirGIDFile, pkgfile.Name(), want, got) + } +} diff --git a/src/cmd/go/internal/work/buildid.go b/src/cmd/go/internal/work/buildid.go new file mode 100644 index 0000000..bf923d0 --- /dev/null +++ b/src/cmd/go/internal/work/buildid.go @@ -0,0 +1,715 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cache" + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/str" + "cmd/internal/buildid" + "cmd/internal/quoted" +) + +// Build IDs +// +// Go packages and binaries are stamped with build IDs that record both +// the action ID, which is a hash of the inputs to the action that produced +// the packages or binary, and the content ID, which is a hash of the action +// output, namely the archive or binary itself. The hash is the same one +// used by the build artifact cache (see cmd/go/internal/cache), but +// truncated when stored in packages and binaries, as the full length is not +// needed and is a bit unwieldy. The precise form is +// +// actionID/[.../]contentID +// +// where the actionID and contentID are prepared by buildid.HashToString below. +// and are found by looking for the first or last slash. +// Usually the buildID is simply actionID/contentID, but see below for an +// exception. +// +// The build ID serves two primary purposes. +// +// 1. The action ID half allows installed packages and binaries to serve as +// one-element cache entries. If we intend to build math.a with a given +// set of inputs summarized in the action ID, and the installed math.a already +// has that action ID, we can reuse the installed math.a instead of rebuilding it. +// +// 2. The content ID half allows the easy preparation of action IDs for steps +// that consume a particular package or binary. The content hash of every +// input file for a given action must be included in the action ID hash. +// Storing the content ID in the build ID lets us read it from the file with +// minimal I/O, instead of reading and hashing the entire file. +// This is especially effective since packages and binaries are typically +// the largest inputs to an action. +// +// Separating action ID from content ID is important for reproducible builds. +// The compiler is compiled with itself. If an output were represented by its +// own action ID (instead of content ID) when computing the action ID of +// the next step in the build process, then the compiler could never have its +// own input action ID as its output action ID (short of a miraculous hash collision). +// Instead we use the content IDs to compute the next action ID, and because +// the content IDs converge, so too do the action IDs and therefore the +// build IDs and the overall compiler binary. See cmd/dist's cmdbootstrap +// for the actual convergence sequence. +// +// The “one-element cache” purpose is a bit more complex for installed +// binaries. For a binary, like cmd/gofmt, there are two steps: compile +// cmd/gofmt/*.go into main.a, and then link main.a into the gofmt binary. +// We do not install gofmt's main.a, only the gofmt binary. Being able to +// decide that the gofmt binary is up-to-date means computing the action ID +// for the final link of the gofmt binary and comparing it against the +// already-installed gofmt binary. But computing the action ID for the link +// means knowing the content ID of main.a, which we did not keep. +// To sidestep this problem, each binary actually stores an expanded build ID: +// +// actionID(binary)/actionID(main.a)/contentID(main.a)/contentID(binary) +// +// (Note that this can be viewed equivalently as: +// +// actionID(binary)/buildID(main.a)/contentID(binary) +// +// Storing the buildID(main.a) in the middle lets the computations that care +// about the prefix or suffix halves ignore the middle and preserves the +// original build ID as a contiguous string.) +// +// During the build, when it's time to build main.a, the gofmt binary has the +// information needed to decide whether the eventual link would produce +// the same binary: if the action ID for main.a's inputs matches and then +// the action ID for the link step matches when assuming the given main.a +// content ID, then the binary as a whole is up-to-date and need not be rebuilt. +// +// This is all a bit complex and may be simplified once we can rely on the +// main cache, but at least at the start we will be using the content-based +// staleness determination without a cache beyond the usual installed +// package and binary locations. + +const buildIDSeparator = "/" + +// actionID returns the action ID half of a build ID. +func actionID(buildID string) string { + i := strings.Index(buildID, buildIDSeparator) + if i < 0 { + return buildID + } + return buildID[:i] +} + +// contentID returns the content ID half of a build ID. +func contentID(buildID string) string { + return buildID[strings.LastIndex(buildID, buildIDSeparator)+1:] +} + +// toolID returns the unique ID to use for the current copy of the +// named tool (asm, compile, cover, link). +// +// It is important that if the tool changes (for example a compiler bug is fixed +// and the compiler reinstalled), toolID returns a different string, so that old +// package archives look stale and are rebuilt (with the fixed compiler). +// This suggests using a content hash of the tool binary, as stored in the build ID. +// +// Unfortunately, we can't just open the tool binary, because the tool might be +// invoked via a wrapper program specified by -toolexec and we don't know +// what the wrapper program does. In particular, we want "-toolexec toolstash" +// to continue working: it does no good if "-toolexec toolstash" is executing a +// stashed copy of the compiler but the go command is acting as if it will run +// the standard copy of the compiler. The solution is to ask the tool binary to tell +// us its own build ID using the "-V=full" flag now supported by all tools. +// Then we know we're getting the build ID of the compiler that will actually run +// during the build. (How does the compiler binary know its own content hash? +// We store it there using updateBuildID after the standard link step.) +// +// A final twist is that we'd prefer to have reproducible builds for release toolchains. +// It should be possible to cross-compile for Windows from either Linux or Mac +// or Windows itself and produce the same binaries, bit for bit. If the tool ID, +// which influences the action ID half of the build ID, is based on the content ID, +// then the Linux compiler binary and Mac compiler binary will have different tool IDs +// and therefore produce executables with different action IDs. +// To avoid this problem, for releases we use the release version string instead +// of the compiler binary's content hash. This assumes that all compilers built +// on all different systems are semantically equivalent, which is of course only true +// modulo bugs. (Producing the exact same executables also requires that the different +// build setups agree on details like $GOROOT and file name paths, but at least the +// tool IDs do not make it impossible.) +func (b *Builder) toolID(name string) string { + b.id.Lock() + id := b.toolIDCache[name] + b.id.Unlock() + + if id != "" { + return id + } + + path := base.Tool(name) + desc := "go tool " + name + + // Special case: undocumented -vettool overrides usual vet, + // for testing vet or supplying an alternative analysis tool. + if name == "vet" && VetTool != "" { + path = VetTool + desc = VetTool + } + + cmdline := str.StringList(cfg.BuildToolexec, path, "-V=full") + cmd := exec.Command(cmdline[0], cmdline[1:]...) + var stdout, stderr strings.Builder + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + if stderr.Len() > 0 { + os.Stderr.WriteString(stderr.String()) + } + base.Fatalf("go: error obtaining buildID for %s: %v", desc, err) + } + + line := stdout.String() + f := strings.Fields(line) + if len(f) < 3 || f[0] != name && path != VetTool || f[1] != "version" || f[2] == "devel" && !strings.HasPrefix(f[len(f)-1], "buildID=") { + base.Fatalf("go: parsing buildID from %s -V=full: unexpected output:\n\t%s", desc, line) + } + if f[2] == "devel" { + // On the development branch, use the content ID part of the build ID. + id = contentID(f[len(f)-1]) + } else { + // For a release, the output is like: "compile version go1.9.1 X:framepointer". + // Use the whole line. + id = strings.TrimSpace(line) + } + + b.id.Lock() + b.toolIDCache[name] = id + b.id.Unlock() + + return id +} + +// gccToolID returns the unique ID to use for a tool that is invoked +// by the GCC driver. This is used particularly for gccgo, but this can also +// be used for gcc, g++, gfortran, etc.; those tools all use the GCC +// driver under different names. The approach used here should also +// work for sufficiently new versions of clang. Unlike toolID, the +// name argument is the program to run. The language argument is the +// type of input file as passed to the GCC driver's -x option. +// +// For these tools we have no -V=full option to dump the build ID, +// but we can run the tool with -v -### to reliably get the compiler proper +// and hash that. That will work in the presence of -toolexec. +// +// In order to get reproducible builds for released compilers, we +// detect a released compiler by the absence of "experimental" in the +// --version output, and in that case we just use the version string. +// +// gccToolID also returns the underlying executable for the compiler. +// The caller assumes that stat of the exe can be used, combined with the id, +// to detect changes in the underlying compiler. The returned exe can be empty, +// which means to rely only on the id. +func (b *Builder) gccToolID(name, language string) (id, exe string, err error) { + key := name + "." + language + b.id.Lock() + id = b.toolIDCache[key] + exe = b.toolIDCache[key+".exe"] + b.id.Unlock() + + if id != "" { + return id, exe, nil + } + + // Invoke the driver with -### to see the subcommands and the + // version strings. Use -x to set the language. Pretend to + // compile an empty file on standard input. + cmdline := str.StringList(cfg.BuildToolexec, name, "-###", "-x", language, "-c", "-") + cmd := exec.Command(cmdline[0], cmdline[1:]...) + // Force untranslated output so that we see the string "version". + cmd.Env = append(os.Environ(), "LC_ALL=C") + out, err := cmd.CombinedOutput() + if err != nil { + return "", "", fmt.Errorf("%s: %v; output: %q", name, err, out) + } + + version := "" + lines := strings.Split(string(out), "\n") + for _, line := range lines { + fields := strings.Fields(line) + for i, field := range fields { + if strings.HasSuffix(field, ":") { + // Avoid parsing fields of lines like "Configured with: …", which may + // contain arbitrary substrings. + break + } + if field == "version" && i < len(fields)-1 { + // Check that the next field is plausibly a version number. + // We require only that it begins with an ASCII digit, + // since we don't know what version numbering schemes a given + // C compiler may use. (Clang and GCC mostly seem to follow the scheme X.Y.Z, + // but in https://go.dev/issue/64619 we saw "8.3 [DragonFly]", and who knows + // what other C compilers like "zig cc" might report?) + next := fields[i+1] + if len(next) > 0 && next[0] >= '0' && next[0] <= '9' { + version = line + break + } + } + } + if version != "" { + break + } + } + if version == "" { + return "", "", fmt.Errorf("%s: can not find version number in %q", name, out) + } + + if !strings.Contains(version, "experimental") { + // This is a release. Use this line as the tool ID. + id = version + } else { + // This is a development version. The first line with + // a leading space is the compiler proper. + compiler := "" + for _, line := range lines { + if strings.HasPrefix(line, " ") && !strings.HasPrefix(line, " (in-process)") { + compiler = line + break + } + } + if compiler == "" { + return "", "", fmt.Errorf("%s: can not find compilation command in %q", name, out) + } + + fields, _ := quoted.Split(compiler) + if len(fields) == 0 { + return "", "", fmt.Errorf("%s: compilation command confusion %q", name, out) + } + exe = fields[0] + if !strings.ContainsAny(exe, `/\`) { + if lp, err := cfg.LookPath(exe); err == nil { + exe = lp + } + } + id, err = buildid.ReadFile(exe) + if err != nil { + return "", "", err + } + + // If we can't find a build ID, use a hash. + if id == "" { + id = b.fileHash(exe) + } + } + + b.id.Lock() + b.toolIDCache[key] = id + b.toolIDCache[key+".exe"] = exe + b.id.Unlock() + + return id, exe, nil +} + +// Check if assembler used by gccgo is GNU as. +func assemblerIsGas() bool { + cmd := exec.Command(BuildToolchain.compiler(), "-print-prog-name=as") + assembler, err := cmd.Output() + if err == nil { + cmd := exec.Command(strings.TrimSpace(string(assembler)), "--version") + out, err := cmd.Output() + return err == nil && strings.Contains(string(out), "GNU") + } else { + return false + } +} + +// gccgoBuildIDFile creates an assembler file that records the +// action's build ID in an SHF_EXCLUDE section for ELF files or +// in a CSECT in XCOFF files. +func (b *Builder) gccgoBuildIDFile(a *Action) (string, error) { + sfile := a.Objdir + "_buildid.s" + + var buf bytes.Buffer + if cfg.Goos == "aix" { + fmt.Fprintf(&buf, "\t.csect .go.buildid[XO]\n") + } else if (cfg.Goos != "solaris" && cfg.Goos != "illumos") || assemblerIsGas() { + fmt.Fprintf(&buf, "\t"+`.section .go.buildid,"e"`+"\n") + } else if cfg.Goarch == "sparc" || cfg.Goarch == "sparc64" { + fmt.Fprintf(&buf, "\t"+`.section ".go.buildid",#exclude`+"\n") + } else { // cfg.Goarch == "386" || cfg.Goarch == "amd64" + fmt.Fprintf(&buf, "\t"+`.section .go.buildid,#exclude`+"\n") + } + fmt.Fprintf(&buf, "\t.byte ") + for i := 0; i < len(a.buildID); i++ { + if i > 0 { + if i%8 == 0 { + fmt.Fprintf(&buf, "\n\t.byte ") + } else { + fmt.Fprintf(&buf, ",") + } + } + fmt.Fprintf(&buf, "%#02x", a.buildID[i]) + } + fmt.Fprintf(&buf, "\n") + if cfg.Goos != "solaris" && cfg.Goos != "illumos" && cfg.Goos != "aix" { + secType := "@progbits" + if cfg.Goarch == "arm" { + secType = "%progbits" + } + fmt.Fprintf(&buf, "\t"+`.section .note.GNU-stack,"",%s`+"\n", secType) + fmt.Fprintf(&buf, "\t"+`.section .note.GNU-split-stack,"",%s`+"\n", secType) + } + + if err := b.Shell(a).writeFile(sfile, buf.Bytes()); err != nil { + return "", err + } + + return sfile, nil +} + +// buildID returns the build ID found in the given file. +// If no build ID is found, buildID returns the content hash of the file. +func (b *Builder) buildID(file string) string { + b.id.Lock() + id := b.buildIDCache[file] + b.id.Unlock() + + if id != "" { + return id + } + + id, err := buildid.ReadFile(file) + if err != nil { + id = b.fileHash(file) + } + + b.id.Lock() + b.buildIDCache[file] = id + b.id.Unlock() + + return id +} + +// fileHash returns the content hash of the named file. +func (b *Builder) fileHash(file string) string { + file, _ = fsys.OverlayPath(file) + sum, err := cache.FileHash(file) + if err != nil { + return "" + } + return buildid.HashToString(sum) +} + +// useCache tries to satisfy the action a, which has action ID actionHash, +// by using a cached result from an earlier build. At the moment, the only +// cached result is the installed package or binary at target. +// If useCache decides that the cache can be used, it sets a.buildID +// and a.built for use by parent actions and then returns true. +// Otherwise it sets a.buildID to a temporary build ID for use in the build +// and returns false. When useCache returns false the expectation is that +// the caller will build the target and then call updateBuildID to finish the +// build ID computation. +// When useCache returns false, it may have initiated buffering of output +// during a's work. The caller should defer b.flushOutput(a), to make sure +// that flushOutput is eventually called regardless of whether the action +// succeeds. The flushOutput call must happen after updateBuildID. +func (b *Builder) useCache(a *Action, actionHash cache.ActionID, target string, printOutput bool) bool { + // The second half of the build ID here is a placeholder for the content hash. + // It's important that the overall buildID be unlikely verging on impossible + // to appear in the output by chance, but that should be taken care of by + // the actionID half; if it also appeared in the input that would be like an + // engineered 120-bit partial SHA256 collision. + a.actionID = actionHash + actionID := buildid.HashToString(actionHash) + if a.json != nil { + a.json.ActionID = actionID + } + contentID := actionID // temporary placeholder, likely unique + a.buildID = actionID + buildIDSeparator + contentID + + // Executable binaries also record the main build ID in the middle. + // See "Build IDs" comment above. + if a.Mode == "link" { + mainpkg := a.Deps[0] + a.buildID = actionID + buildIDSeparator + mainpkg.buildID + buildIDSeparator + contentID + } + + // If user requested -a, we force a rebuild, so don't use the cache. + if cfg.BuildA { + if p := a.Package; p != nil && !p.Stale { + p.Stale = true + p.StaleReason = "build -a flag in use" + } + // Begin saving output for later writing to cache. + a.output = []byte{} + return false + } + + c := cache.Default() + + if target != "" { + buildID, _ := buildid.ReadFile(target) + if strings.HasPrefix(buildID, actionID+buildIDSeparator) { + a.buildID = buildID + if a.json != nil { + a.json.BuildID = a.buildID + } + a.built = target + // Poison a.Target to catch uses later in the build. + a.Target = "DO NOT USE - " + a.Mode + return true + } + // Special case for building a main package: if the only thing we + // want the package for is to link a binary, and the binary is + // already up-to-date, then to avoid a rebuild, report the package + // as up-to-date as well. See "Build IDs" comment above. + // TODO(rsc): Rewrite this code to use a TryCache func on the link action. + if !b.NeedExport && a.Mode == "build" && len(a.triggers) == 1 && a.triggers[0].Mode == "link" { + if id := strings.Split(buildID, buildIDSeparator); len(id) == 4 && id[1] == actionID { + // Temporarily assume a.buildID is the package build ID + // stored in the installed binary, and see if that makes + // the upcoming link action ID a match. If so, report that + // we built the package, safe in the knowledge that the + // link step will not ask us for the actual package file. + // Note that (*Builder).LinkAction arranged that all of + // a.triggers[0]'s dependencies other than a are also + // dependencies of a, so that we can be sure that, + // other than a.buildID, b.linkActionID is only accessing + // build IDs of completed actions. + oldBuildID := a.buildID + a.buildID = id[1] + buildIDSeparator + id[2] + linkID := buildid.HashToString(b.linkActionID(a.triggers[0])) + if id[0] == linkID { + // Best effort attempt to display output from the compile and link steps. + // If it doesn't work, it doesn't work: reusing the cached binary is more + // important than reprinting diagnostic information. + if printOutput { + showStdout(b, c, a, "stdout") // compile output + showStdout(b, c, a, "link-stdout") // link output + } + + // Poison a.Target to catch uses later in the build. + a.Target = "DO NOT USE - main build pseudo-cache Target" + a.built = "DO NOT USE - main build pseudo-cache built" + if a.json != nil { + a.json.BuildID = a.buildID + } + return true + } + // Otherwise restore old build ID for main build. + a.buildID = oldBuildID + } + } + } + + // Special case for linking a test binary: if the only thing we + // want the binary for is to run the test, and the test result is cached, + // then to avoid the link step, report the link as up-to-date. + // We avoid the nested build ID problem in the previous special case + // by recording the test results in the cache under the action ID half. + if len(a.triggers) == 1 && a.triggers[0].TryCache != nil && a.triggers[0].TryCache(b, a.triggers[0]) { + // Best effort attempt to display output from the compile and link steps. + // If it doesn't work, it doesn't work: reusing the test result is more + // important than reprinting diagnostic information. + if printOutput { + showStdout(b, c, a.Deps[0], "stdout") // compile output + showStdout(b, c, a.Deps[0], "link-stdout") // link output + } + + // Poison a.Target to catch uses later in the build. + a.Target = "DO NOT USE - pseudo-cache Target" + a.built = "DO NOT USE - pseudo-cache built" + return true + } + + // Check to see if the action output is cached. + if file, _, err := cache.GetFile(c, actionHash); err == nil { + if buildID, err := buildid.ReadFile(file); err == nil { + if printOutput { + showStdout(b, c, a, "stdout") + } + a.built = file + a.Target = "DO NOT USE - using cache" + a.buildID = buildID + if a.json != nil { + a.json.BuildID = a.buildID + } + if p := a.Package; p != nil && target != "" { + p.Stale = true + // Clearer than explaining that something else is stale. + p.StaleReason = "not installed but available in build cache" + } + return true + } + } + + // If we've reached this point, we can't use the cache for the action. + if p := a.Package; p != nil && !p.Stale { + p.Stale = true + p.StaleReason = "build ID mismatch" + if b.IsCmdList { + // Since we may end up printing StaleReason, include more detail. + for _, p1 := range p.Internal.Imports { + if p1.Stale && p1.StaleReason != "" { + if strings.HasPrefix(p1.StaleReason, "stale dependency: ") { + p.StaleReason = p1.StaleReason + break + } + if strings.HasPrefix(p.StaleReason, "build ID mismatch") { + p.StaleReason = "stale dependency: " + p1.ImportPath + } + } + } + } + } + + // Begin saving output for later writing to cache. + a.output = []byte{} + return false +} + +func showStdout(b *Builder, c cache.Cache, a *Action, key string) error { + actionID := a.actionID + + stdout, stdoutEntry, err := cache.GetBytes(c, cache.Subkey(actionID, key)) + if err != nil { + return err + } + + if len(stdout) > 0 { + sh := b.Shell(a) + if cfg.BuildX || cfg.BuildN { + sh.ShowCmd("", "%s # internal", joinUnambiguously(str.StringList("cat", c.OutputFile(stdoutEntry.OutputID)))) + } + if !cfg.BuildN { + sh.Print(string(stdout)) + } + } + return nil +} + +// flushOutput flushes the output being queued in a. +func (b *Builder) flushOutput(a *Action) { + b.Shell(a).Print(string(a.output)) + a.output = nil +} + +// updateBuildID updates the build ID in the target written by action a. +// It requires that useCache was called for action a and returned false, +// and that the build was then carried out and given the temporary +// a.buildID to record as the build ID in the resulting package or binary. +// updateBuildID computes the final content ID and updates the build IDs +// in the binary. +// +// Keep in sync with src/cmd/buildid/buildid.go +func (b *Builder) updateBuildID(a *Action, target string, rewrite bool) error { + sh := b.Shell(a) + + if cfg.BuildX || cfg.BuildN { + if rewrite { + sh.ShowCmd("", "%s # internal", joinUnambiguously(str.StringList(base.Tool("buildid"), "-w", target))) + } + if cfg.BuildN { + return nil + } + } + + c := cache.Default() + + // Cache output from compile/link, even if we don't do the rest. + switch a.Mode { + case "build": + cache.PutBytes(c, cache.Subkey(a.actionID, "stdout"), a.output) + case "link": + // Even though we don't cache the binary, cache the linker text output. + // We might notice that an installed binary is up-to-date but still + // want to pretend to have run the linker. + // Store it under the main package's action ID + // to make it easier to find when that's all we have. + for _, a1 := range a.Deps { + if p1 := a1.Package; p1 != nil && p1.Name == "main" { + cache.PutBytes(c, cache.Subkey(a1.actionID, "link-stdout"), a.output) + break + } + } + } + + // Find occurrences of old ID and compute new content-based ID. + r, err := os.Open(target) + if err != nil { + return err + } + matches, hash, err := buildid.FindAndHash(r, a.buildID, 0) + r.Close() + if err != nil { + return err + } + newID := a.buildID[:strings.LastIndex(a.buildID, buildIDSeparator)] + buildIDSeparator + buildid.HashToString(hash) + if len(newID) != len(a.buildID) { + return fmt.Errorf("internal error: build ID length mismatch %q vs %q", a.buildID, newID) + } + + // Replace with new content-based ID. + a.buildID = newID + if a.json != nil { + a.json.BuildID = a.buildID + } + if len(matches) == 0 { + // Assume the user specified -buildid= to override what we were going to choose. + return nil + } + + if rewrite { + w, err := os.OpenFile(target, os.O_RDWR, 0) + if err != nil { + return err + } + err = buildid.Rewrite(w, matches, newID) + if err != nil { + w.Close() + return err + } + if err := w.Close(); err != nil { + return err + } + } + + // Cache package builds, but not binaries (link steps). + // The expectation is that binaries are not reused + // nearly as often as individual packages, and they're + // much larger, so the cache-footprint-to-utility ratio + // of binaries is much lower for binaries. + // Not caching the link step also makes sure that repeated "go run" at least + // always rerun the linker, so that they don't get too fast. + // (We don't want people thinking go is a scripting language.) + // Note also that if we start caching binaries, then we will + // copy the binaries out of the cache to run them, and then + // that will mean the go process is itself writing a binary + // and then executing it, so we will need to defend against + // ETXTBSY problems as discussed in exec.go and golang.org/issue/22220. + if a.Mode == "build" { + r, err := os.Open(target) + if err == nil { + if a.output == nil { + panic("internal error: a.output not set") + } + outputID, _, err := c.Put(a.actionID, r) + r.Close() + if err == nil && cfg.BuildX { + sh.ShowCmd("", "%s # internal", joinUnambiguously(str.StringList("cp", target, c.OutputFile(outputID)))) + } + if b.NeedExport { + if err != nil { + return err + } + a.Package.Export = c.OutputFile(outputID) + a.Package.BuildID = a.buildID + } + } + } + + return nil +} diff --git a/src/cmd/go/internal/work/cover.go b/src/cmd/go/internal/work/cover.go new file mode 100644 index 0000000..c0acc61 --- /dev/null +++ b/src/cmd/go/internal/work/cover.go @@ -0,0 +1,150 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Action graph execution methods related to coverage. + +package work + +import ( + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/str" + "cmd/internal/cov/covcmd" + "context" + "encoding/json" + "fmt" + "internal/coverage" + "io" + "os" + "path/filepath" +) + +// CovData invokes "go tool covdata" with the specified arguments +// as part of the execution of action 'a'. +func (b *Builder) CovData(a *Action, cmdargs ...any) ([]byte, error) { + cmdline := str.StringList(cmdargs...) + args := append([]string{}, cfg.BuildToolexec...) + args = append(args, base.Tool("covdata")) + args = append(args, cmdline...) + return b.Shell(a).runOut(a.Objdir, nil, args) +} + +// BuildActionCoverMetaFile locates and returns the path of the +// meta-data file written by the "go tool cover" step as part of the +// build action for the "go test -cover" run action 'runAct'. Note +// that if the package has no functions the meta-data file will exist +// but will be empty; in this case the return is an empty string. +func BuildActionCoverMetaFile(runAct *Action) (string, error) { + p := runAct.Package + for i := range runAct.Deps { + pred := runAct.Deps[i] + if pred.Mode != "build" || pred.Package == nil { + continue + } + if pred.Package.ImportPath == p.ImportPath { + metaFile := pred.Objdir + covcmd.MetaFileForPackage(p.ImportPath) + f, err := os.Open(metaFile) + if err != nil { + return "", err + } + defer f.Close() + fi, err2 := f.Stat() + if err2 != nil { + return "", err2 + } + if fi.Size() == 0 { + return "", nil + } + return metaFile, nil + } + } + return "", fmt.Errorf("internal error: unable to locate build action for package %q run action", p.ImportPath) +} + +// WriteCoveragePercent writes out to the writer 'w' a "percent +// statements covered" for the package whose test-run action is +// 'runAct', based on the meta-data file 'mf'. This helper is used in +// cases where a user runs "go test -cover" on a package that has +// functions but no tests; in the normal case (package has tests) +// the percentage is written by the test binary when it runs. +func WriteCoveragePercent(b *Builder, runAct *Action, mf string, w io.Writer) error { + dir := filepath.Dir(mf) + output, cerr := b.CovData(runAct, "percent", "-i", dir) + if cerr != nil { + return b.Shell(runAct).reportCmd("", "", output, cerr) + } + _, werr := w.Write(output) + return werr +} + +// WriteCoverageProfile writes out a coverage profile fragment for the +// package whose test-run action is 'runAct'; content is written to +// the file 'outf' based on the coverage meta-data info found in +// 'mf'. This helper is used in cases where a user runs "go test +// -cover" on a package that has functions but no tests. +func WriteCoverageProfile(b *Builder, runAct *Action, mf, outf string, w io.Writer) error { + dir := filepath.Dir(mf) + output, err := b.CovData(runAct, "textfmt", "-i", dir, "-o", outf) + if err != nil { + return b.Shell(runAct).reportCmd("", "", output, err) + } + _, werr := w.Write(output) + return werr +} + +// WriteCoverMetaFilesFile writes out a summary file ("meta-files +// file") as part of the action function for the "writeCoverMeta" +// pseudo action employed during "go test -coverpkg" runs where there +// are multiple tests and multiple packages covered. It builds up a +// table mapping package import path to meta-data file fragment and +// writes it out to a file where it can be read by the various test +// run actions. Note that this function has to be called A) after the +// build actions are complete for all packages being tested, and B) +// before any of the "run test" actions for those packages happen. +// This requirement is enforced by adding making this action ("a") +// dependent on all test package build actions, and making all test +// run actions dependent on this action. +func WriteCoverMetaFilesFile(b *Builder, ctx context.Context, a *Action) error { + sh := b.Shell(a) + + // Build the metafilecollection object. + var collection coverage.MetaFileCollection + for i := range a.Deps { + dep := a.Deps[i] + if dep.Mode != "build" { + panic("unexpected mode " + dep.Mode) + } + metaFilesFile := dep.Objdir + covcmd.MetaFileForPackage(dep.Package.ImportPath) + // Check to make sure the meta-data file fragment exists + // and has content (may be empty if package has no functions). + if fi, err := os.Stat(metaFilesFile); err != nil { + continue + } else if fi.Size() == 0 { + continue + } + collection.ImportPaths = append(collection.ImportPaths, dep.Package.ImportPath) + collection.MetaFileFragments = append(collection.MetaFileFragments, metaFilesFile) + } + + // Serialize it. + data, err := json.Marshal(collection) + if err != nil { + return fmt.Errorf("marshal MetaFileCollection: %v", err) + } + data = append(data, '\n') // makes -x output more readable + + // Create the directory for this action's objdir and + // then write out the serialized collection + // to a file in the directory. + if err := sh.Mkdir(a.Objdir); err != nil { + return err + } + mfpath := a.Objdir + coverage.MetaFilesFileName + if err := sh.writeFile(mfpath, data); err != nil { + return fmt.Errorf("writing metafiles file: %v", err) + } + + // We're done. + return nil +} diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go new file mode 100644 index 0000000..e05471b --- /dev/null +++ b/src/cmd/go/internal/work/exec.go @@ -0,0 +1,3472 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Action graph execution. + +package work + +import ( + "bytes" + "cmd/internal/cov/covcmd" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "go/token" + "internal/lazyregexp" + "io" + "io/fs" + "log" + "math/rand" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "slices" + "sort" + "strconv" + "strings" + "sync" + "time" + + "cmd/go/internal/base" + "cmd/go/internal/cache" + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/gover" + "cmd/go/internal/load" + "cmd/go/internal/modload" + "cmd/go/internal/str" + "cmd/go/internal/trace" + "cmd/internal/buildid" + "cmd/internal/quoted" + "cmd/internal/sys" +) + +const defaultCFlags = "-O2 -g" + +// actionList returns the list of actions in the dag rooted at root +// as visited in a depth-first post-order traversal. +func actionList(root *Action) []*Action { + seen := map[*Action]bool{} + all := []*Action{} + var walk func(*Action) + walk = func(a *Action) { + if seen[a] { + return + } + seen[a] = true + for _, a1 := range a.Deps { + walk(a1) + } + all = append(all, a) + } + walk(root) + return all +} + +// Do runs the action graph rooted at root. +func (b *Builder) Do(ctx context.Context, root *Action) { + ctx, span := trace.StartSpan(ctx, "exec.Builder.Do ("+root.Mode+" "+root.Target+")") + defer span.Done() + + if !b.IsCmdList { + // If we're doing real work, take time at the end to trim the cache. + c := cache.Default() + defer func() { + if err := c.Close(); err != nil { + base.Fatalf("go: failed to trim cache: %v", err) + } + }() + } + + // Build list of all actions, assigning depth-first post-order priority. + // The original implementation here was a true queue + // (using a channel) but it had the effect of getting + // distracted by low-level leaf actions to the detriment + // of completing higher-level actions. The order of + // work does not matter much to overall execution time, + // but when running "go test std" it is nice to see each test + // results as soon as possible. The priorities assigned + // ensure that, all else being equal, the execution prefers + // to do what it would have done first in a simple depth-first + // dependency order traversal. + all := actionList(root) + for i, a := range all { + a.priority = i + } + + // Write action graph, without timing information, in case we fail and exit early. + writeActionGraph := func() { + if file := cfg.DebugActiongraph; file != "" { + if strings.HasSuffix(file, ".go") { + // Do not overwrite Go source code in: + // go build -debug-actiongraph x.go + base.Fatalf("go: refusing to write action graph to %v\n", file) + } + js := actionGraphJSON(root) + if err := os.WriteFile(file, []byte(js), 0666); err != nil { + fmt.Fprintf(os.Stderr, "go: writing action graph: %v\n", err) + base.SetExitStatus(1) + } + } + } + writeActionGraph() + + b.readySema = make(chan bool, len(all)) + + // Initialize per-action execution state. + for _, a := range all { + for _, a1 := range a.Deps { + a1.triggers = append(a1.triggers, a) + } + a.pending = len(a.Deps) + if a.pending == 0 { + b.ready.push(a) + b.readySema <- true + } + } + + // Handle runs a single action and takes care of triggering + // any actions that are runnable as a result. + handle := func(ctx context.Context, a *Action) { + if a.json != nil { + a.json.TimeStart = time.Now() + } + var err error + if a.Actor != nil && (!a.Failed || a.IgnoreFail) { + // TODO(matloob): Better action descriptions + desc := "Executing action " + if a.Package != nil { + desc += "(" + a.Mode + " " + a.Package.Desc() + ")" + } + ctx, span := trace.StartSpan(ctx, desc) + a.traceSpan = span + for _, d := range a.Deps { + trace.Flow(ctx, d.traceSpan, a.traceSpan) + } + err = a.Actor.Act(b, ctx, a) + span.Done() + } + if a.json != nil { + a.json.TimeDone = time.Now() + } + + // The actions run in parallel but all the updates to the + // shared work state are serialized through b.exec. + b.exec.Lock() + defer b.exec.Unlock() + + if err != nil { + if b.AllowErrors && a.Package != nil { + if a.Package.Error == nil { + a.Package.Error = &load.PackageError{Err: err} + a.Package.Incomplete = true + } + } else { + var ipe load.ImportPathError + if a.Package != nil && (!errors.As(err, &ipe) || ipe.ImportPath() != a.Package.ImportPath) { + err = fmt.Errorf("%s: %v", a.Package.ImportPath, err) + } + base.Errorf("%s", err) + } + a.Failed = true + } + + for _, a0 := range a.triggers { + if a.Failed { + a0.Failed = true + } + if a0.pending--; a0.pending == 0 { + b.ready.push(a0) + b.readySema <- true + } + } + + if a == root { + close(b.readySema) + } + } + + var wg sync.WaitGroup + + // Kick off goroutines according to parallelism. + // If we are using the -n flag (just printing commands) + // drop the parallelism to 1, both to make the output + // deterministic and because there is no real work anyway. + par := cfg.BuildP + if cfg.BuildN { + par = 1 + } + for i := 0; i < par; i++ { + wg.Add(1) + go func() { + ctx := trace.StartGoroutine(ctx) + defer wg.Done() + for { + select { + case _, ok := <-b.readySema: + if !ok { + return + } + // Receiving a value from b.readySema entitles + // us to take from the ready queue. + b.exec.Lock() + a := b.ready.pop() + b.exec.Unlock() + handle(ctx, a) + case <-base.Interrupted: + base.SetExitStatus(1) + return + } + } + }() + } + + wg.Wait() + + // Write action graph again, this time with timing information. + writeActionGraph() +} + +// buildActionID computes the action ID for a build action. +func (b *Builder) buildActionID(a *Action) cache.ActionID { + p := a.Package + h := cache.NewHash("build " + p.ImportPath) + + // Configuration independent of compiler toolchain. + // Note: buildmode has already been accounted for in buildGcflags + // and should not be inserted explicitly. Most buildmodes use the + // same compiler settings and can reuse each other's results. + // If not, the reason is already recorded in buildGcflags. + fmt.Fprintf(h, "compile\n") + + // Include information about the origin of the package that + // may be embedded in the debug info for the object file. + if cfg.BuildTrimpath { + // When -trimpath is used with a package built from the module cache, + // its debug information refers to the module path and version + // instead of the directory. + if p.Module != nil { + fmt.Fprintf(h, "module %s@%s\n", p.Module.Path, p.Module.Version) + } + } else if p.Goroot { + // The Go compiler always hides the exact value of $GOROOT + // when building things in GOROOT. + // + // The C compiler does not, but for packages in GOROOT we rewrite the path + // as though -trimpath were set, so that we don't invalidate the build cache + // (and especially any precompiled C archive files) when changing + // GOROOT_FINAL. (See https://go.dev/issue/50183.) + // + // b.WorkDir is always either trimmed or rewritten to + // the literal string "/tmp/go-build". + } else if !strings.HasPrefix(p.Dir, b.WorkDir) { + // -trimpath is not set and no other rewrite rules apply, + // so the object file may refer to the absolute directory + // containing the package. + fmt.Fprintf(h, "dir %s\n", p.Dir) + } + + if p.Module != nil { + fmt.Fprintf(h, "go %s\n", p.Module.GoVersion) + } + fmt.Fprintf(h, "goos %s goarch %s\n", cfg.Goos, cfg.Goarch) + fmt.Fprintf(h, "import %q\n", p.ImportPath) + fmt.Fprintf(h, "omitdebug %v standard %v local %v prefix %q\n", p.Internal.OmitDebug, p.Standard, p.Internal.Local, p.Internal.LocalPrefix) + if cfg.BuildTrimpath { + fmt.Fprintln(h, "trimpath") + } + if p.Internal.ForceLibrary { + fmt.Fprintf(h, "forcelibrary\n") + } + if len(p.CgoFiles)+len(p.SwigFiles)+len(p.SwigCXXFiles) > 0 { + fmt.Fprintf(h, "cgo %q\n", b.toolID("cgo")) + cppflags, cflags, cxxflags, fflags, ldflags, _ := b.CFlags(p) + + ccExe := b.ccExe() + fmt.Fprintf(h, "CC=%q %q %q %q\n", ccExe, cppflags, cflags, ldflags) + // Include the C compiler tool ID so that if the C + // compiler changes we rebuild the package. + if ccID, _, err := b.gccToolID(ccExe[0], "c"); err == nil { + fmt.Fprintf(h, "CC ID=%q\n", ccID) + } + if len(p.CXXFiles)+len(p.SwigCXXFiles) > 0 { + cxxExe := b.cxxExe() + fmt.Fprintf(h, "CXX=%q %q\n", cxxExe, cxxflags) + if cxxID, _, err := b.gccToolID(cxxExe[0], "c++"); err == nil { + fmt.Fprintf(h, "CXX ID=%q\n", cxxID) + } + } + if len(p.FFiles) > 0 { + fcExe := b.fcExe() + fmt.Fprintf(h, "FC=%q %q\n", fcExe, fflags) + if fcID, _, err := b.gccToolID(fcExe[0], "f95"); err == nil { + fmt.Fprintf(h, "FC ID=%q\n", fcID) + } + } + // TODO(rsc): Should we include the SWIG version? + } + if p.Internal.Cover.Mode != "" { + fmt.Fprintf(h, "cover %q %q\n", p.Internal.Cover.Mode, b.toolID("cover")) + } + if p.Internal.FuzzInstrument { + if fuzzFlags := fuzzInstrumentFlags(); fuzzFlags != nil { + fmt.Fprintf(h, "fuzz %q\n", fuzzFlags) + } + } + if p.Internal.BuildInfo != nil { + fmt.Fprintf(h, "modinfo %q\n", p.Internal.BuildInfo.String()) + } + + // Configuration specific to compiler toolchain. + switch cfg.BuildToolchainName { + default: + base.Fatalf("buildActionID: unknown build toolchain %q", cfg.BuildToolchainName) + case "gc": + fmt.Fprintf(h, "compile %s %q %q\n", b.toolID("compile"), forcedGcflags, p.Internal.Gcflags) + if len(p.SFiles) > 0 { + fmt.Fprintf(h, "asm %q %q %q\n", b.toolID("asm"), forcedAsmflags, p.Internal.Asmflags) + } + + // GOARM, GOMIPS, etc. + key, val := cfg.GetArchEnv() + fmt.Fprintf(h, "%s=%s\n", key, val) + + if cfg.CleanGOEXPERIMENT != "" { + fmt.Fprintf(h, "GOEXPERIMENT=%q\n", cfg.CleanGOEXPERIMENT) + } + + // TODO(rsc): Convince compiler team not to add more magic environment variables, + // or perhaps restrict the environment variables passed to subprocesses. + // Because these are clumsy, undocumented special-case hacks + // for debugging the compiler, they are not settable using 'go env -w', + // and so here we use os.Getenv, not cfg.Getenv. + magic := []string{ + "GOCLOBBERDEADHASH", + "GOSSAFUNC", + "GOSSADIR", + "GOCOMPILEDEBUG", + } + for _, env := range magic { + if x := os.Getenv(env); x != "" { + fmt.Fprintf(h, "magic %s=%s\n", env, x) + } + } + + case "gccgo": + id, _, err := b.gccToolID(BuildToolchain.compiler(), "go") + if err != nil { + base.Fatalf("%v", err) + } + fmt.Fprintf(h, "compile %s %q %q\n", id, forcedGccgoflags, p.Internal.Gccgoflags) + fmt.Fprintf(h, "pkgpath %s\n", gccgoPkgpath(p)) + fmt.Fprintf(h, "ar %q\n", BuildToolchain.(gccgoToolchain).ar()) + if len(p.SFiles) > 0 { + id, _, _ = b.gccToolID(BuildToolchain.compiler(), "assembler-with-cpp") + // Ignore error; different assembler versions + // are unlikely to make any difference anyhow. + fmt.Fprintf(h, "asm %q\n", id) + } + } + + // Input files. + inputFiles := str.StringList( + p.GoFiles, + p.CgoFiles, + p.CFiles, + p.CXXFiles, + p.FFiles, + p.MFiles, + p.HFiles, + p.SFiles, + p.SysoFiles, + p.SwigFiles, + p.SwigCXXFiles, + p.EmbedFiles, + ) + for _, file := range inputFiles { + fmt.Fprintf(h, "file %s %s\n", file, b.fileHash(filepath.Join(p.Dir, file))) + } + if p.Internal.PGOProfile != "" { + fmt.Fprintf(h, "pgofile %s\n", b.fileHash(p.Internal.PGOProfile)) + } + for _, a1 := range a.Deps { + p1 := a1.Package + if p1 != nil { + fmt.Fprintf(h, "import %s %s\n", p1.ImportPath, contentID(a1.buildID)) + } + } + + return h.Sum() +} + +// needCgoHdr reports whether the actions triggered by this one +// expect to be able to access the cgo-generated header file. +func (b *Builder) needCgoHdr(a *Action) bool { + // If this build triggers a header install, run cgo to get the header. + if !b.IsCmdList && (a.Package.UsesCgo() || a.Package.UsesSwig()) && (cfg.BuildBuildmode == "c-archive" || cfg.BuildBuildmode == "c-shared") { + for _, t1 := range a.triggers { + if t1.Mode == "install header" { + return true + } + } + for _, t1 := range a.triggers { + for _, t2 := range t1.triggers { + if t2.Mode == "install header" { + return true + } + } + } + } + return false +} + +// allowedVersion reports whether the version v is an allowed version of go +// (one that we can compile). +// v is known to be of the form "1.23". +func allowedVersion(v string) bool { + // Special case: no requirement. + if v == "" { + return true + } + return gover.Compare(gover.Local(), v) >= 0 +} + +const ( + needBuild uint32 = 1 << iota + needCgoHdr + needVet + needCompiledGoFiles + needCovMetaFile + needStale +) + +// build is the action for building a single package. +// Note that any new influence on this logic must be reported in b.buildActionID above as well. +func (b *Builder) build(ctx context.Context, a *Action) (err error) { + p := a.Package + sh := b.Shell(a) + + bit := func(x uint32, b bool) uint32 { + if b { + return x + } + return 0 + } + + cachedBuild := false + needCovMeta := p.Internal.Cover.GenMeta + need := bit(needBuild, !b.IsCmdList && a.needBuild || b.NeedExport) | + bit(needCgoHdr, b.needCgoHdr(a)) | + bit(needVet, a.needVet) | + bit(needCovMetaFile, needCovMeta) | + bit(needCompiledGoFiles, b.NeedCompiledGoFiles) + + if !p.BinaryOnly { + if b.useCache(a, b.buildActionID(a), p.Target, need&needBuild != 0) { + // We found the main output in the cache. + // If we don't need any other outputs, we can stop. + // Otherwise, we need to write files to a.Objdir (needVet, needCgoHdr). + // Remember that we might have them in cache + // and check again after we create a.Objdir. + cachedBuild = true + a.output = []byte{} // start saving output in case we miss any cache results + need &^= needBuild + if b.NeedExport { + p.Export = a.built + p.BuildID = a.buildID + } + if need&needCompiledGoFiles != 0 { + if err := b.loadCachedCompiledGoFiles(a); err == nil { + need &^= needCompiledGoFiles + } + } + } + + // Source files might be cached, even if the full action is not + // (e.g., go list -compiled -find). + if !cachedBuild && need&needCompiledGoFiles != 0 { + if err := b.loadCachedCompiledGoFiles(a); err == nil { + need &^= needCompiledGoFiles + } + } + + if need == 0 { + return nil + } + defer b.flushOutput(a) + } + + defer func() { + if err != nil && b.IsCmdList && b.NeedError && p.Error == nil { + p.Error = &load.PackageError{Err: err} + } + }() + if cfg.BuildN { + // In -n mode, print a banner between packages. + // The banner is five lines so that when changes to + // different sections of the bootstrap script have to + // be merged, the banners give patch something + // to use to find its context. + sh.Print("\n#\n# " + p.ImportPath + "\n#\n\n") + } + + if cfg.BuildV { + sh.Print(p.ImportPath + "\n") + } + + if p.Error != nil { + // Don't try to build anything for packages with errors. There may be a + // problem with the inputs that makes the package unsafe to build. + return p.Error + } + + if p.BinaryOnly { + p.Stale = true + p.StaleReason = "binary-only packages are no longer supported" + if b.IsCmdList { + return nil + } + return errors.New("binary-only packages are no longer supported") + } + + if p.Module != nil && !allowedVersion(p.Module.GoVersion) { + return errors.New("module requires Go " + p.Module.GoVersion + " or later") + } + + if err := b.checkDirectives(a); err != nil { + return err + } + + if err := sh.Mkdir(a.Objdir); err != nil { + return err + } + objdir := a.Objdir + + // Load cached cgo header, but only if we're skipping the main build (cachedBuild==true). + if cachedBuild && need&needCgoHdr != 0 { + if err := b.loadCachedCgoHdr(a); err == nil { + need &^= needCgoHdr + } + } + + // Load cached coverage meta-data file fragment, but only if we're + // skipping the main build (cachedBuild==true). + if cachedBuild && need&needCovMetaFile != 0 { + bact := a.Actor.(*buildActor) + if err := b.loadCachedObjdirFile(a, cache.Default(), bact.covMetaFileName); err == nil { + need &^= needCovMetaFile + } + } + + // Load cached vet config, but only if that's all we have left + // (need == needVet, not testing just the one bit). + // If we are going to do a full build anyway, + // we're going to regenerate the files below anyway. + if need == needVet { + if err := b.loadCachedVet(a); err == nil { + need &^= needVet + } + } + if need == 0 { + return nil + } + + if err := AllowInstall(a); err != nil { + return err + } + + // make target directory + dir, _ := filepath.Split(a.Target) + if dir != "" { + if err := sh.Mkdir(dir); err != nil { + return err + } + } + + gofiles := str.StringList(p.GoFiles) + cgofiles := str.StringList(p.CgoFiles) + cfiles := str.StringList(p.CFiles) + sfiles := str.StringList(p.SFiles) + cxxfiles := str.StringList(p.CXXFiles) + var objects, cgoObjects, pcCFLAGS, pcLDFLAGS []string + + if p.UsesCgo() || p.UsesSwig() { + if pcCFLAGS, pcLDFLAGS, err = b.getPkgConfigFlags(a); err != nil { + return + } + } + + // Compute overlays for .c/.cc/.h/etc. and if there are any overlays + // put correct contents of all those files in the objdir, to ensure + // the correct headers are included. nonGoOverlay is the overlay that + // points from nongo files to the copied files in objdir. + nonGoFileLists := [][]string{p.CFiles, p.SFiles, p.CXXFiles, p.HFiles, p.FFiles} +OverlayLoop: + for _, fs := range nonGoFileLists { + for _, f := range fs { + if _, ok := fsys.OverlayPath(mkAbs(p.Dir, f)); ok { + a.nonGoOverlay = make(map[string]string) + break OverlayLoop + } + } + } + if a.nonGoOverlay != nil { + for _, fs := range nonGoFileLists { + for i := range fs { + from := mkAbs(p.Dir, fs[i]) + opath, _ := fsys.OverlayPath(from) + dst := objdir + filepath.Base(fs[i]) + if err := sh.CopyFile(dst, opath, 0666, false); err != nil { + return err + } + a.nonGoOverlay[from] = dst + } + } + } + + // If we're doing coverage, preprocess the .go files and put them in the work directory + if p.Internal.Cover.Mode != "" { + outfiles := []string{} + infiles := []string{} + for i, file := range str.StringList(gofiles, cgofiles) { + if base.IsTestFile(file) { + continue // Not covering this file. + } + + var sourceFile string + var coverFile string + var key string + if base, found := strings.CutSuffix(file, ".cgo1.go"); found { + // cgo files have absolute paths + base = filepath.Base(base) + sourceFile = file + coverFile = objdir + base + ".cgo1.go" + key = base + ".go" + } else { + sourceFile = filepath.Join(p.Dir, file) + coverFile = objdir + file + key = file + } + coverFile = strings.TrimSuffix(coverFile, ".go") + ".cover.go" + if cfg.Experiment.CoverageRedesign { + infiles = append(infiles, sourceFile) + outfiles = append(outfiles, coverFile) + } else { + cover := p.Internal.CoverVars[key] + if cover == nil { + continue // Not covering this file. + } + if err := b.cover(a, coverFile, sourceFile, cover.Var); err != nil { + return err + } + } + if i < len(gofiles) { + gofiles[i] = coverFile + } else { + cgofiles[i-len(gofiles)] = coverFile + } + } + + if cfg.Experiment.CoverageRedesign { + if len(infiles) != 0 { + // Coverage instrumentation creates new top level + // variables in the target package for things like + // meta-data containers, counter vars, etc. To avoid + // collisions with user variables, suffix the var name + // with 12 hex digits from the SHA-256 hash of the + // import path. Choice of 12 digits is historical/arbitrary, + // we just need enough of the hash to avoid accidents, + // as opposed to precluding determined attempts by + // users to break things. + sum := sha256.Sum256([]byte(a.Package.ImportPath)) + coverVar := fmt.Sprintf("goCover_%x_", sum[:6]) + mode := a.Package.Internal.Cover.Mode + if mode == "" { + panic("covermode should be set at this point") + } + if newoutfiles, err := b.cover2(a, infiles, outfiles, coverVar, mode); err != nil { + return err + } else { + outfiles = newoutfiles + gofiles = append([]string{newoutfiles[0]}, gofiles...) + } + } else { + // If there are no input files passed to cmd/cover, + // then we don't want to pass -covercfg when building + // the package with the compiler, so set covermode to + // the empty string so as to signal that we need to do + // that. + p.Internal.Cover.Mode = "" + } + if ba, ok := a.Actor.(*buildActor); ok && ba.covMetaFileName != "" { + b.cacheObjdirFile(a, cache.Default(), ba.covMetaFileName) + } + } + } + + // Run SWIG on each .swig and .swigcxx file. + // Each run will generate two files, a .go file and a .c or .cxx file. + // The .go file will use import "C" and is to be processed by cgo. + // For -cover test or build runs, this needs to happen after the cover + // tool is run; we don't want to instrument swig-generated Go files, + // see issue #64661. + if p.UsesSwig() { + outGo, outC, outCXX, err := b.swig(a, objdir, pcCFLAGS) + if err != nil { + return err + } + cgofiles = append(cgofiles, outGo...) + cfiles = append(cfiles, outC...) + cxxfiles = append(cxxfiles, outCXX...) + } + + // Run cgo. + if p.UsesCgo() || p.UsesSwig() { + // In a package using cgo, cgo compiles the C, C++ and assembly files with gcc. + // There is one exception: runtime/cgo's job is to bridge the + // cgo and non-cgo worlds, so it necessarily has files in both. + // In that case gcc only gets the gcc_* files. + var gccfiles []string + gccfiles = append(gccfiles, cfiles...) + cfiles = nil + if p.Standard && p.ImportPath == "runtime/cgo" { + filter := func(files, nongcc, gcc []string) ([]string, []string) { + for _, f := range files { + if strings.HasPrefix(f, "gcc_") { + gcc = append(gcc, f) + } else { + nongcc = append(nongcc, f) + } + } + return nongcc, gcc + } + sfiles, gccfiles = filter(sfiles, sfiles[:0], gccfiles) + } else { + for _, sfile := range sfiles { + data, err := os.ReadFile(filepath.Join(p.Dir, sfile)) + if err == nil { + if bytes.HasPrefix(data, []byte("TEXT")) || bytes.Contains(data, []byte("\nTEXT")) || + bytes.HasPrefix(data, []byte("DATA")) || bytes.Contains(data, []byte("\nDATA")) || + bytes.HasPrefix(data, []byte("GLOBL")) || bytes.Contains(data, []byte("\nGLOBL")) { + return fmt.Errorf("package using cgo has Go assembly file %s", sfile) + } + } + } + gccfiles = append(gccfiles, sfiles...) + sfiles = nil + } + + outGo, outObj, err := b.cgo(a, base.Tool("cgo"), objdir, pcCFLAGS, pcLDFLAGS, mkAbsFiles(p.Dir, cgofiles), gccfiles, cxxfiles, p.MFiles, p.FFiles) + + // The files in cxxfiles have now been handled by b.cgo. + cxxfiles = nil + + if err != nil { + return err + } + if cfg.BuildToolchainName == "gccgo" { + cgoObjects = append(cgoObjects, a.Objdir+"_cgo_flags") + } + cgoObjects = append(cgoObjects, outObj...) + gofiles = append(gofiles, outGo...) + + switch cfg.BuildBuildmode { + case "c-archive", "c-shared": + b.cacheCgoHdr(a) + } + } + + var srcfiles []string // .go and non-.go + srcfiles = append(srcfiles, gofiles...) + srcfiles = append(srcfiles, sfiles...) + srcfiles = append(srcfiles, cfiles...) + srcfiles = append(srcfiles, cxxfiles...) + b.cacheSrcFiles(a, srcfiles) + + // Running cgo generated the cgo header. + need &^= needCgoHdr + + // Sanity check only, since Package.load already checked as well. + if len(gofiles) == 0 { + return &load.NoGoError{Package: p} + } + + // Prepare Go vet config if needed. + if need&needVet != 0 { + buildVetConfig(a, srcfiles) + need &^= needVet + } + if need&needCompiledGoFiles != 0 { + if err := b.loadCachedCompiledGoFiles(a); err != nil { + return fmt.Errorf("loading compiled Go files from cache: %w", err) + } + need &^= needCompiledGoFiles + } + if need == 0 { + // Nothing left to do. + return nil + } + + // Collect symbol ABI requirements from assembly. + symabis, err := BuildToolchain.symabis(b, a, sfiles) + if err != nil { + return err + } + + // Prepare Go import config. + // We start it off with a comment so it can't be empty, so icfg.Bytes() below is never nil. + // It should never be empty anyway, but there have been bugs in the past that resulted + // in empty configs, which then unfortunately turn into "no config passed to compiler", + // and the compiler falls back to looking in pkg itself, which mostly works, + // except when it doesn't. + var icfg bytes.Buffer + fmt.Fprintf(&icfg, "# import config\n") + for i, raw := range p.Internal.RawImports { + final := p.Imports[i] + if final != raw { + fmt.Fprintf(&icfg, "importmap %s=%s\n", raw, final) + } + } + for _, a1 := range a.Deps { + p1 := a1.Package + if p1 == nil || p1.ImportPath == "" || a1.built == "" { + continue + } + fmt.Fprintf(&icfg, "packagefile %s=%s\n", p1.ImportPath, a1.built) + } + + // Prepare Go embed config if needed. + // Unlike the import config, it's okay for the embed config to be empty. + var embedcfg []byte + if len(p.Internal.Embed) > 0 { + var embed struct { + Patterns map[string][]string + Files map[string]string + } + embed.Patterns = p.Internal.Embed + embed.Files = make(map[string]string) + for _, file := range p.EmbedFiles { + embed.Files[file] = filepath.Join(p.Dir, file) + } + js, err := json.MarshalIndent(&embed, "", "\t") + if err != nil { + return fmt.Errorf("marshal embedcfg: %v", err) + } + embedcfg = js + } + + if p.Internal.BuildInfo != nil && cfg.ModulesEnabled { + prog := modload.ModInfoProg(p.Internal.BuildInfo.String(), cfg.BuildToolchainName == "gccgo") + if len(prog) > 0 { + if err := sh.writeFile(objdir+"_gomod_.go", prog); err != nil { + return err + } + gofiles = append(gofiles, objdir+"_gomod_.go") + } + } + + // Compile Go. + objpkg := objdir + "_pkg_.a" + ofile, out, err := BuildToolchain.gc(b, a, objpkg, icfg.Bytes(), embedcfg, symabis, len(sfiles) > 0, gofiles) + if err := sh.reportCmd("", "", out, err); err != nil { + return err + } + if ofile != objpkg { + objects = append(objects, ofile) + } + + // Copy .h files named for goos or goarch or goos_goarch + // to names using GOOS and GOARCH. + // For example, defs_linux_amd64.h becomes defs_GOOS_GOARCH.h. + _goos_goarch := "_" + cfg.Goos + "_" + cfg.Goarch + _goos := "_" + cfg.Goos + _goarch := "_" + cfg.Goarch + for _, file := range p.HFiles { + name, ext := fileExtSplit(file) + switch { + case strings.HasSuffix(name, _goos_goarch): + targ := file[:len(name)-len(_goos_goarch)] + "_GOOS_GOARCH." + ext + if err := sh.CopyFile(objdir+targ, filepath.Join(p.Dir, file), 0666, true); err != nil { + return err + } + case strings.HasSuffix(name, _goarch): + targ := file[:len(name)-len(_goarch)] + "_GOARCH." + ext + if err := sh.CopyFile(objdir+targ, filepath.Join(p.Dir, file), 0666, true); err != nil { + return err + } + case strings.HasSuffix(name, _goos): + targ := file[:len(name)-len(_goos)] + "_GOOS." + ext + if err := sh.CopyFile(objdir+targ, filepath.Join(p.Dir, file), 0666, true); err != nil { + return err + } + } + } + + for _, file := range cfiles { + out := file[:len(file)-len(".c")] + ".o" + if err := BuildToolchain.cc(b, a, objdir+out, file); err != nil { + return err + } + objects = append(objects, out) + } + + // Assemble .s files. + if len(sfiles) > 0 { + ofiles, err := BuildToolchain.asm(b, a, sfiles) + if err != nil { + return err + } + objects = append(objects, ofiles...) + } + + // For gccgo on ELF systems, we write the build ID as an assembler file. + // This lets us set the SHF_EXCLUDE flag. + // This is read by readGccgoArchive in cmd/internal/buildid/buildid.go. + if a.buildID != "" && cfg.BuildToolchainName == "gccgo" { + switch cfg.Goos { + case "aix", "android", "dragonfly", "freebsd", "illumos", "linux", "netbsd", "openbsd", "solaris": + asmfile, err := b.gccgoBuildIDFile(a) + if err != nil { + return err + } + ofiles, err := BuildToolchain.asm(b, a, []string{asmfile}) + if err != nil { + return err + } + objects = append(objects, ofiles...) + } + } + + // NOTE(rsc): On Windows, it is critically important that the + // gcc-compiled objects (cgoObjects) be listed after the ordinary + // objects in the archive. I do not know why this is. + // https://golang.org/issue/2601 + objects = append(objects, cgoObjects...) + + // Add system object files. + for _, syso := range p.SysoFiles { + objects = append(objects, filepath.Join(p.Dir, syso)) + } + + // Pack into archive in objdir directory. + // If the Go compiler wrote an archive, we only need to add the + // object files for non-Go sources to the archive. + // If the Go compiler wrote an archive and the package is entirely + // Go sources, there is no pack to execute at all. + if len(objects) > 0 { + if err := BuildToolchain.pack(b, a, objpkg, objects); err != nil { + return err + } + } + + if err := b.updateBuildID(a, objpkg, true); err != nil { + return err + } + + a.built = objpkg + return nil +} + +func (b *Builder) checkDirectives(a *Action) error { + var msg *bytes.Buffer + p := a.Package + var seen map[string]token.Position + for _, d := range p.Internal.Build.Directives { + if strings.HasPrefix(d.Text, "//go:debug") { + key, _, err := load.ParseGoDebug(d.Text) + if err != nil && err != load.ErrNotGoDebug { + if msg == nil { + msg = new(bytes.Buffer) + } + fmt.Fprintf(msg, "%s: invalid //go:debug: %v\n", d.Pos, err) + continue + } + if pos, ok := seen[key]; ok { + fmt.Fprintf(msg, "%s: repeated //go:debug for %v\n\t%s: previous //go:debug\n", d.Pos, key, pos) + continue + } + if seen == nil { + seen = make(map[string]token.Position) + } + seen[key] = d.Pos + } + } + if msg != nil { + // We pass a non-nil error to reportCmd to trigger the failure reporting + // path, but the content of the error doesn't matter because msg is + // non-empty. + err := errors.New("invalid directive") + return b.Shell(a).reportCmd("", "", msg.Bytes(), err) + } + return nil +} + +func (b *Builder) cacheObjdirFile(a *Action, c cache.Cache, name string) error { + f, err := os.Open(a.Objdir + name) + if err != nil { + return err + } + defer f.Close() + _, _, err = c.Put(cache.Subkey(a.actionID, name), f) + return err +} + +func (b *Builder) findCachedObjdirFile(a *Action, c cache.Cache, name string) (string, error) { + file, _, err := cache.GetFile(c, cache.Subkey(a.actionID, name)) + if err != nil { + return "", fmt.Errorf("loading cached file %s: %w", name, err) + } + return file, nil +} + +func (b *Builder) loadCachedObjdirFile(a *Action, c cache.Cache, name string) error { + cached, err := b.findCachedObjdirFile(a, c, name) + if err != nil { + return err + } + return b.Shell(a).CopyFile(a.Objdir+name, cached, 0666, true) +} + +func (b *Builder) cacheCgoHdr(a *Action) { + c := cache.Default() + b.cacheObjdirFile(a, c, "_cgo_install.h") +} + +func (b *Builder) loadCachedCgoHdr(a *Action) error { + c := cache.Default() + return b.loadCachedObjdirFile(a, c, "_cgo_install.h") +} + +func (b *Builder) cacheSrcFiles(a *Action, srcfiles []string) { + c := cache.Default() + var buf bytes.Buffer + for _, file := range srcfiles { + if !strings.HasPrefix(file, a.Objdir) { + // not generated + buf.WriteString("./") + buf.WriteString(file) + buf.WriteString("\n") + continue + } + name := file[len(a.Objdir):] + buf.WriteString(name) + buf.WriteString("\n") + if err := b.cacheObjdirFile(a, c, name); err != nil { + return + } + } + cache.PutBytes(c, cache.Subkey(a.actionID, "srcfiles"), buf.Bytes()) +} + +func (b *Builder) loadCachedVet(a *Action) error { + c := cache.Default() + list, _, err := cache.GetBytes(c, cache.Subkey(a.actionID, "srcfiles")) + if err != nil { + return fmt.Errorf("reading srcfiles list: %w", err) + } + var srcfiles []string + for _, name := range strings.Split(string(list), "\n") { + if name == "" { // end of list + continue + } + if strings.HasPrefix(name, "./") { + srcfiles = append(srcfiles, name[2:]) + continue + } + if err := b.loadCachedObjdirFile(a, c, name); err != nil { + return err + } + srcfiles = append(srcfiles, a.Objdir+name) + } + buildVetConfig(a, srcfiles) + return nil +} + +func (b *Builder) loadCachedCompiledGoFiles(a *Action) error { + c := cache.Default() + list, _, err := cache.GetBytes(c, cache.Subkey(a.actionID, "srcfiles")) + if err != nil { + return fmt.Errorf("reading srcfiles list: %w", err) + } + var gofiles []string + for _, name := range strings.Split(string(list), "\n") { + if name == "" { // end of list + continue + } else if !strings.HasSuffix(name, ".go") { + continue + } + if strings.HasPrefix(name, "./") { + gofiles = append(gofiles, name[len("./"):]) + continue + } + file, err := b.findCachedObjdirFile(a, c, name) + if err != nil { + return fmt.Errorf("finding %s: %w", name, err) + } + gofiles = append(gofiles, file) + } + a.Package.CompiledGoFiles = gofiles + return nil +} + +// vetConfig is the configuration passed to vet describing a single package. +type vetConfig struct { + ID string // package ID (example: "fmt [fmt.test]") + Compiler string // compiler name (gc, gccgo) + Dir string // directory containing package + ImportPath string // canonical import path ("package path") + GoFiles []string // absolute paths to package source files + NonGoFiles []string // absolute paths to package non-Go files + IgnoredFiles []string // absolute paths to ignored source files + + ImportMap map[string]string // map import path in source code to package path + PackageFile map[string]string // map package path to .a file with export data + Standard map[string]bool // map package path to whether it's in the standard library + PackageVetx map[string]string // map package path to vetx data from earlier vet run + VetxOnly bool // only compute vetx data; don't report detected problems + VetxOutput string // write vetx data to this output file + GoVersion string // Go version for package + + SucceedOnTypecheckFailure bool // awful hack; see #18395 and below +} + +func buildVetConfig(a *Action, srcfiles []string) { + // Classify files based on .go extension. + // srcfiles does not include raw cgo files. + var gofiles, nongofiles []string + for _, name := range srcfiles { + if strings.HasSuffix(name, ".go") { + gofiles = append(gofiles, name) + } else { + nongofiles = append(nongofiles, name) + } + } + + ignored := str.StringList(a.Package.IgnoredGoFiles, a.Package.IgnoredOtherFiles) + + // Pass list of absolute paths to vet, + // so that vet's error messages will use absolute paths, + // so that we can reformat them relative to the directory + // in which the go command is invoked. + vcfg := &vetConfig{ + ID: a.Package.ImportPath, + Compiler: cfg.BuildToolchainName, + Dir: a.Package.Dir, + GoFiles: mkAbsFiles(a.Package.Dir, gofiles), + NonGoFiles: mkAbsFiles(a.Package.Dir, nongofiles), + IgnoredFiles: mkAbsFiles(a.Package.Dir, ignored), + ImportPath: a.Package.ImportPath, + ImportMap: make(map[string]string), + PackageFile: make(map[string]string), + Standard: make(map[string]bool), + } + if a.Package.Module != nil { + v := a.Package.Module.GoVersion + if v == "" { + v = gover.DefaultGoModVersion + } + vcfg.GoVersion = "go" + v + } + a.vetCfg = vcfg + for i, raw := range a.Package.Internal.RawImports { + final := a.Package.Imports[i] + vcfg.ImportMap[raw] = final + } + + // Compute the list of mapped imports in the vet config + // so that we can add any missing mappings below. + vcfgMapped := make(map[string]bool) + for _, p := range vcfg.ImportMap { + vcfgMapped[p] = true + } + + for _, a1 := range a.Deps { + p1 := a1.Package + if p1 == nil || p1.ImportPath == "" { + continue + } + // Add import mapping if needed + // (for imports like "runtime/cgo" that appear only in generated code). + if !vcfgMapped[p1.ImportPath] { + vcfg.ImportMap[p1.ImportPath] = p1.ImportPath + } + if a1.built != "" { + vcfg.PackageFile[p1.ImportPath] = a1.built + } + if p1.Standard { + vcfg.Standard[p1.ImportPath] = true + } + } +} + +// VetTool is the path to an alternate vet tool binary. +// The caller is expected to set it (if needed) before executing any vet actions. +var VetTool string + +// VetFlags are the default flags to pass to vet. +// The caller is expected to set them before executing any vet actions. +var VetFlags []string + +// VetExplicit records whether the vet flags were set explicitly on the command line. +var VetExplicit bool + +func (b *Builder) vet(ctx context.Context, a *Action) error { + // a.Deps[0] is the build of the package being vetted. + // a.Deps[1] is the build of the "fmt" package. + + a.Failed = false // vet of dependency may have failed but we can still succeed + + if a.Deps[0].Failed { + // The build of the package has failed. Skip vet check. + // Vet could return export data for non-typecheck errors, + // but we ignore it because the package cannot be compiled. + return nil + } + + vcfg := a.Deps[0].vetCfg + if vcfg == nil { + // Vet config should only be missing if the build failed. + return fmt.Errorf("vet config not found") + } + + sh := b.Shell(a) + + vcfg.VetxOnly = a.VetxOnly + vcfg.VetxOutput = a.Objdir + "vet.out" + vcfg.PackageVetx = make(map[string]string) + + h := cache.NewHash("vet " + a.Package.ImportPath) + fmt.Fprintf(h, "vet %q\n", b.toolID("vet")) + + vetFlags := VetFlags + + // In GOROOT, we enable all the vet tests during 'go test', + // not just the high-confidence subset. This gets us extra + // checking for the standard library (at some compliance cost) + // and helps us gain experience about how well the checks + // work, to help decide which should be turned on by default. + // The command-line still wins. + // + // Note that this flag change applies even when running vet as + // a dependency of vetting a package outside std. + // (Otherwise we'd have to introduce a whole separate + // space of "vet fmt as a dependency of a std top-level vet" + // versus "vet fmt as a dependency of a non-std top-level vet".) + // This is OK as long as the packages that are farther down the + // dependency tree turn on *more* analysis, as here. + // (The unsafeptr check does not write any facts for use by + // later vet runs, nor does unreachable.) + if a.Package.Goroot && !VetExplicit && VetTool == "" { + // Turn off -unsafeptr checks. + // There's too much unsafe.Pointer code + // that vet doesn't like in low-level packages + // like runtime, sync, and reflect. + // Note that $GOROOT/src/buildall.bash + // does the same + // and should be updated if these flags are + // changed here. + vetFlags = []string{"-unsafeptr=false"} + + // Also turn off -unreachable checks during go test. + // During testing it is very common to make changes + // like hard-coded forced returns or panics that make + // code unreachable. It's unreasonable to insist on files + // not having any unreachable code during "go test". + // (buildall.bash still has -unreachable enabled + // for the overall whole-tree scan.) + if cfg.CmdName == "test" { + vetFlags = append(vetFlags, "-unreachable=false") + } + } + + // Note: We could decide that vet should compute export data for + // all analyses, in which case we don't need to include the flags here. + // But that would mean that if an analysis causes problems like + // unexpected crashes there would be no way to turn it off. + // It seems better to let the flags disable export analysis too. + fmt.Fprintf(h, "vetflags %q\n", vetFlags) + + fmt.Fprintf(h, "pkg %q\n", a.Deps[0].actionID) + for _, a1 := range a.Deps { + if a1.Mode == "vet" && a1.built != "" { + fmt.Fprintf(h, "vetout %q %s\n", a1.Package.ImportPath, b.fileHash(a1.built)) + vcfg.PackageVetx[a1.Package.ImportPath] = a1.built + } + } + key := cache.ActionID(h.Sum()) + + if vcfg.VetxOnly && !cfg.BuildA { + c := cache.Default() + if file, _, err := cache.GetFile(c, key); err == nil { + a.built = file + return nil + } + } + + js, err := json.MarshalIndent(vcfg, "", "\t") + if err != nil { + return fmt.Errorf("internal error marshaling vet config: %v", err) + } + js = append(js, '\n') + if err := sh.writeFile(a.Objdir+"vet.cfg", js); err != nil { + return err + } + + // TODO(rsc): Why do we pass $GCCGO to go vet? + env := b.cCompilerEnv() + if cfg.BuildToolchainName == "gccgo" { + env = append(env, "GCCGO="+BuildToolchain.compiler()) + } + + p := a.Package + tool := VetTool + if tool == "" { + tool = base.Tool("vet") + } + runErr := sh.run(p.Dir, p.ImportPath, env, cfg.BuildToolexec, tool, vetFlags, a.Objdir+"vet.cfg") + + // If vet wrote export data, save it for input to future vets. + if f, err := os.Open(vcfg.VetxOutput); err == nil { + a.built = vcfg.VetxOutput + cache.Default().Put(key, f) + f.Close() + } + + return runErr +} + +// linkActionID computes the action ID for a link action. +func (b *Builder) linkActionID(a *Action) cache.ActionID { + p := a.Package + h := cache.NewHash("link " + p.ImportPath) + + // Toolchain-independent configuration. + fmt.Fprintf(h, "link\n") + fmt.Fprintf(h, "buildmode %s goos %s goarch %s\n", cfg.BuildBuildmode, cfg.Goos, cfg.Goarch) + fmt.Fprintf(h, "import %q\n", p.ImportPath) + fmt.Fprintf(h, "omitdebug %v standard %v local %v prefix %q\n", p.Internal.OmitDebug, p.Standard, p.Internal.Local, p.Internal.LocalPrefix) + if cfg.BuildTrimpath { + fmt.Fprintln(h, "trimpath") + } + + // Toolchain-dependent configuration, shared with b.linkSharedActionID. + b.printLinkerConfig(h, p) + + // Input files. + for _, a1 := range a.Deps { + p1 := a1.Package + if p1 != nil { + if a1.built != "" || a1.buildID != "" { + buildID := a1.buildID + if buildID == "" { + buildID = b.buildID(a1.built) + } + fmt.Fprintf(h, "packagefile %s=%s\n", p1.ImportPath, contentID(buildID)) + } + // Because we put package main's full action ID into the binary's build ID, + // we must also put the full action ID into the binary's action ID hash. + if p1.Name == "main" { + fmt.Fprintf(h, "packagemain %s\n", a1.buildID) + } + if p1.Shlib != "" { + fmt.Fprintf(h, "packageshlib %s=%s\n", p1.ImportPath, contentID(b.buildID(p1.Shlib))) + } + } + } + + return h.Sum() +} + +// printLinkerConfig prints the linker config into the hash h, +// as part of the computation of a linker-related action ID. +func (b *Builder) printLinkerConfig(h io.Writer, p *load.Package) { + switch cfg.BuildToolchainName { + default: + base.Fatalf("linkActionID: unknown toolchain %q", cfg.BuildToolchainName) + + case "gc": + fmt.Fprintf(h, "link %s %q %s\n", b.toolID("link"), forcedLdflags, ldBuildmode) + if p != nil { + fmt.Fprintf(h, "linkflags %q\n", p.Internal.Ldflags) + } + + // GOARM, GOMIPS, etc. + key, val := cfg.GetArchEnv() + fmt.Fprintf(h, "%s=%s\n", key, val) + + if cfg.CleanGOEXPERIMENT != "" { + fmt.Fprintf(h, "GOEXPERIMENT=%q\n", cfg.CleanGOEXPERIMENT) + } + + // The linker writes source file paths that say GOROOT_FINAL, but + // only if -trimpath is not specified (see ld() in gc.go). + gorootFinal := cfg.GOROOT_FINAL + if cfg.BuildTrimpath { + gorootFinal = trimPathGoRootFinal + } + fmt.Fprintf(h, "GOROOT=%s\n", gorootFinal) + + // GO_EXTLINK_ENABLED controls whether the external linker is used. + fmt.Fprintf(h, "GO_EXTLINK_ENABLED=%s\n", cfg.Getenv("GO_EXTLINK_ENABLED")) + + // TODO(rsc): Do cgo settings and flags need to be included? + // Or external linker settings and flags? + + case "gccgo": + id, _, err := b.gccToolID(BuildToolchain.linker(), "go") + if err != nil { + base.Fatalf("%v", err) + } + fmt.Fprintf(h, "link %s %s\n", id, ldBuildmode) + // TODO(iant): Should probably include cgo flags here. + } +} + +// link is the action for linking a single command. +// Note that any new influence on this logic must be reported in b.linkActionID above as well. +func (b *Builder) link(ctx context.Context, a *Action) (err error) { + if b.useCache(a, b.linkActionID(a), a.Package.Target, !b.IsCmdList) || b.IsCmdList { + return nil + } + defer b.flushOutput(a) + + sh := b.Shell(a) + if err := sh.Mkdir(a.Objdir); err != nil { + return err + } + + importcfg := a.Objdir + "importcfg.link" + if err := b.writeLinkImportcfg(a, importcfg); err != nil { + return err + } + + if err := AllowInstall(a); err != nil { + return err + } + + // make target directory + dir, _ := filepath.Split(a.Target) + if dir != "" { + if err := sh.Mkdir(dir); err != nil { + return err + } + } + + if err := BuildToolchain.ld(b, a, a.Target, importcfg, a.Deps[0].built); err != nil { + return err + } + + // Update the binary with the final build ID. + // But if OmitDebug is set, don't rewrite the binary, because we set OmitDebug + // on binaries that we are going to run and then delete. + // There's no point in doing work on such a binary. + // Worse, opening the binary for write here makes it + // essentially impossible to safely fork+exec due to a fundamental + // incompatibility between ETXTBSY and threads on modern Unix systems. + // See golang.org/issue/22220. + // We still call updateBuildID to update a.buildID, which is important + // for test result caching, but passing rewrite=false (final arg) + // means we don't actually rewrite the binary, nor store the + // result into the cache. That's probably a net win: + // less cache space wasted on large binaries we are not likely to + // need again. (On the other hand it does make repeated go test slower.) + // It also makes repeated go run slower, which is a win in itself: + // we don't want people to treat go run like a scripting environment. + if err := b.updateBuildID(a, a.Target, !a.Package.Internal.OmitDebug); err != nil { + return err + } + + a.built = a.Target + return nil +} + +func (b *Builder) writeLinkImportcfg(a *Action, file string) error { + // Prepare Go import cfg. + var icfg bytes.Buffer + for _, a1 := range a.Deps { + p1 := a1.Package + if p1 == nil { + continue + } + fmt.Fprintf(&icfg, "packagefile %s=%s\n", p1.ImportPath, a1.built) + if p1.Shlib != "" { + fmt.Fprintf(&icfg, "packageshlib %s=%s\n", p1.ImportPath, p1.Shlib) + } + } + info := "" + if a.Package.Internal.BuildInfo != nil { + info = a.Package.Internal.BuildInfo.String() + } + fmt.Fprintf(&icfg, "modinfo %q\n", modload.ModInfoData(info)) + return b.Shell(a).writeFile(file, icfg.Bytes()) +} + +// PkgconfigCmd returns a pkg-config binary name +// defaultPkgConfig is defined in zdefaultcc.go, written by cmd/dist. +func (b *Builder) PkgconfigCmd() string { + return envList("PKG_CONFIG", cfg.DefaultPkgConfig)[0] +} + +// splitPkgConfigOutput parses the pkg-config output into a slice of flags. +// This implements the shell quoting semantics described in +// https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_02, +// except that it does not support parameter or arithmetic expansion or command +// substitution and hard-codes the <blank> delimiters instead of reading them +// from LC_LOCALE. +func splitPkgConfigOutput(out []byte) ([]string, error) { + if len(out) == 0 { + return nil, nil + } + var flags []string + flag := make([]byte, 0, len(out)) + didQuote := false // was the current flag parsed from a quoted string? + escaped := false // did we just read `\` in a non-single-quoted context? + quote := byte(0) // what is the quote character around the current string? + + for _, c := range out { + if escaped { + if quote == '"' { + // “The <backslash> shall retain its special meaning as an escape + // character … only when followed by one of the following characters + // when considered special:” + switch c { + case '$', '`', '"', '\\', '\n': + // Handle the escaped character normally. + default: + // Not an escape character after all. + flag = append(flag, '\\', c) + escaped = false + continue + } + } + + if c == '\n' { + // “If a <newline> follows the <backslash>, the shell shall interpret + // this as line continuation.” + } else { + flag = append(flag, c) + } + escaped = false + continue + } + + if quote != 0 && c == quote { + quote = 0 + continue + } + switch quote { + case '\'': + // “preserve the literal value of each character” + flag = append(flag, c) + continue + case '"': + // “preserve the literal value of all characters within the double-quotes, + // with the exception of …” + switch c { + case '`', '$', '\\': + default: + flag = append(flag, c) + continue + } + } + + // “The application shall quote the following characters if they are to + // represent themselves:” + switch c { + case '|', '&', ';', '<', '>', '(', ')', '$', '`': + return nil, fmt.Errorf("unexpected shell character %q in pkgconf output", c) + + case '\\': + // “A <backslash> that is not quoted shall preserve the literal value of + // the following character, with the exception of a <newline>.” + escaped = true + continue + + case '"', '\'': + quote = c + didQuote = true + continue + + case ' ', '\t', '\n': + if len(flag) > 0 || didQuote { + flags = append(flags, string(flag)) + } + flag, didQuote = flag[:0], false + continue + } + + flag = append(flag, c) + } + + // Prefer to report a missing quote instead of a missing escape. If the string + // is something like `"foo\`, it's ambiguous as to whether the trailing + // backslash is really an escape at all. + if quote != 0 { + return nil, errors.New("unterminated quoted string in pkgconf output") + } + if escaped { + return nil, errors.New("broken character escaping in pkgconf output") + } + + if len(flag) > 0 || didQuote { + flags = append(flags, string(flag)) + } + return flags, nil +} + +// Calls pkg-config if needed and returns the cflags/ldflags needed to build a's package. +func (b *Builder) getPkgConfigFlags(a *Action) (cflags, ldflags []string, err error) { + p := a.Package + sh := b.Shell(a) + if pcargs := p.CgoPkgConfig; len(pcargs) > 0 { + // pkg-config permits arguments to appear anywhere in + // the command line. Move them all to the front, before --. + var pcflags []string + var pkgs []string + for _, pcarg := range pcargs { + if pcarg == "--" { + // We're going to add our own "--" argument. + } else if strings.HasPrefix(pcarg, "--") { + pcflags = append(pcflags, pcarg) + } else { + pkgs = append(pkgs, pcarg) + } + } + for _, pkg := range pkgs { + if !load.SafeArg(pkg) { + return nil, nil, fmt.Errorf("invalid pkg-config package name: %s", pkg) + } + } + var out []byte + out, err = sh.runOut(p.Dir, nil, b.PkgconfigCmd(), "--cflags", pcflags, "--", pkgs) + if err != nil { + desc := b.PkgconfigCmd() + " --cflags " + strings.Join(pcflags, " ") + " -- " + strings.Join(pkgs, " ") + return nil, nil, sh.reportCmd(desc, "", out, err) + } + if len(out) > 0 { + cflags, err = splitPkgConfigOutput(bytes.TrimSpace(out)) + if err != nil { + return nil, nil, err + } + if err := checkCompilerFlags("CFLAGS", "pkg-config --cflags", cflags); err != nil { + return nil, nil, err + } + } + out, err = sh.runOut(p.Dir, nil, b.PkgconfigCmd(), "--libs", pcflags, "--", pkgs) + if err != nil { + desc := b.PkgconfigCmd() + " --libs " + strings.Join(pcflags, " ") + " -- " + strings.Join(pkgs, " ") + return nil, nil, sh.reportCmd(desc, "", out, err) + } + if len(out) > 0 { + // We need to handle path with spaces so that C:/Program\ Files can pass + // checkLinkerFlags. Use splitPkgConfigOutput here just like we treat cflags. + ldflags, err = splitPkgConfigOutput(bytes.TrimSpace(out)) + if err != nil { + return nil, nil, err + } + if err := checkLinkerFlags("LDFLAGS", "pkg-config --libs", ldflags); err != nil { + return nil, nil, err + } + } + } + + return +} + +func (b *Builder) installShlibname(ctx context.Context, a *Action) error { + if err := AllowInstall(a); err != nil { + return err + } + + sh := b.Shell(a) + a1 := a.Deps[0] + if !cfg.BuildN { + if err := sh.Mkdir(filepath.Dir(a.Target)); err != nil { + return err + } + } + return sh.writeFile(a.Target, []byte(filepath.Base(a1.Target)+"\n")) +} + +func (b *Builder) linkSharedActionID(a *Action) cache.ActionID { + h := cache.NewHash("linkShared") + + // Toolchain-independent configuration. + fmt.Fprintf(h, "linkShared\n") + fmt.Fprintf(h, "goos %s goarch %s\n", cfg.Goos, cfg.Goarch) + + // Toolchain-dependent configuration, shared with b.linkActionID. + b.printLinkerConfig(h, nil) + + // Input files. + for _, a1 := range a.Deps { + p1 := a1.Package + if a1.built == "" { + continue + } + if p1 != nil { + fmt.Fprintf(h, "packagefile %s=%s\n", p1.ImportPath, contentID(b.buildID(a1.built))) + if p1.Shlib != "" { + fmt.Fprintf(h, "packageshlib %s=%s\n", p1.ImportPath, contentID(b.buildID(p1.Shlib))) + } + } + } + // Files named on command line are special. + for _, a1 := range a.Deps[0].Deps { + p1 := a1.Package + fmt.Fprintf(h, "top %s=%s\n", p1.ImportPath, contentID(b.buildID(a1.built))) + } + + return h.Sum() +} + +func (b *Builder) linkShared(ctx context.Context, a *Action) (err error) { + if b.useCache(a, b.linkSharedActionID(a), a.Target, !b.IsCmdList) || b.IsCmdList { + return nil + } + defer b.flushOutput(a) + + if err := AllowInstall(a); err != nil { + return err + } + + if err := b.Shell(a).Mkdir(a.Objdir); err != nil { + return err + } + + importcfg := a.Objdir + "importcfg.link" + if err := b.writeLinkImportcfg(a, importcfg); err != nil { + return err + } + + // TODO(rsc): There is a missing updateBuildID here, + // but we have to decide where to store the build ID in these files. + a.built = a.Target + return BuildToolchain.ldShared(b, a, a.Deps[0].Deps, a.Target, importcfg, a.Deps) +} + +// BuildInstallFunc is the action for installing a single package or executable. +func BuildInstallFunc(b *Builder, ctx context.Context, a *Action) (err error) { + defer func() { + if err != nil { + // a.Package == nil is possible for the go install -buildmode=shared + // action that installs libmangledname.so, which corresponds to + // a list of packages, not just one. + sep, path := "", "" + if a.Package != nil { + sep, path = " ", a.Package.ImportPath + } + err = fmt.Errorf("go %s%s%s: %v", cfg.CmdName, sep, path, err) + } + }() + sh := b.Shell(a) + + a1 := a.Deps[0] + a.buildID = a1.buildID + if a.json != nil { + a.json.BuildID = a.buildID + } + + // If we are using the eventual install target as an up-to-date + // cached copy of the thing we built, then there's no need to + // copy it into itself (and that would probably fail anyway). + // In this case a1.built == a.Target because a1.built == p.Target, + // so the built target is not in the a1.Objdir tree that b.cleanup(a1) removes. + if a1.built == a.Target { + a.built = a.Target + if !a.buggyInstall { + b.cleanup(a1) + } + // Whether we're smart enough to avoid a complete rebuild + // depends on exactly what the staleness and rebuild algorithms + // are, as well as potentially the state of the Go build cache. + // We don't really want users to be able to infer (or worse start depending on) + // those details from whether the modification time changes during + // "go install", so do a best-effort update of the file times to make it + // look like we rewrote a.Target even if we did not. Updating the mtime + // may also help other mtime-based systems that depend on our + // previous mtime updates that happened more often. + // This is still not perfect - we ignore the error result, and if the file was + // unwritable for some reason then pretending to have written it is also + // confusing - but it's probably better than not doing the mtime update. + // + // But don't do that for the special case where building an executable + // with -linkshared implicitly installs all its dependent libraries. + // We want to hide that awful detail as much as possible, so don't + // advertise it by touching the mtimes (usually the libraries are up + // to date). + if !a.buggyInstall && !b.IsCmdList { + if cfg.BuildN { + sh.ShowCmd("", "touch %s", a.Target) + } else if err := AllowInstall(a); err == nil { + now := time.Now() + os.Chtimes(a.Target, now, now) + } + } + return nil + } + + // If we're building for go list -export, + // never install anything; just keep the cache reference. + if b.IsCmdList { + a.built = a1.built + return nil + } + if err := AllowInstall(a); err != nil { + return err + } + + if err := sh.Mkdir(a.Objdir); err != nil { + return err + } + + perm := fs.FileMode(0666) + if a1.Mode == "link" { + switch cfg.BuildBuildmode { + case "c-archive", "c-shared", "plugin": + default: + perm = 0777 + } + } + + // make target directory + dir, _ := filepath.Split(a.Target) + if dir != "" { + if err := sh.Mkdir(dir); err != nil { + return err + } + } + + if !a.buggyInstall { + defer b.cleanup(a1) + } + + return sh.moveOrCopyFile(a.Target, a1.built, perm, false) +} + +// AllowInstall returns a non-nil error if this invocation of the go command is +// allowed to install a.Target. +// +// The build of cmd/go running under its own test is forbidden from installing +// to its original GOROOT. The var is exported so it can be set by TestMain. +var AllowInstall = func(*Action) error { return nil } + +// cleanup removes a's object dir to keep the amount of +// on-disk garbage down in a large build. On an operating system +// with aggressive buffering, cleaning incrementally like +// this keeps the intermediate objects from hitting the disk. +func (b *Builder) cleanup(a *Action) { + if !cfg.BuildWork { + b.Shell(a).RemoveAll(a.Objdir) + } +} + +// Install the cgo export header file, if there is one. +func (b *Builder) installHeader(ctx context.Context, a *Action) error { + sh := b.Shell(a) + + src := a.Objdir + "_cgo_install.h" + if _, err := os.Stat(src); os.IsNotExist(err) { + // If the file does not exist, there are no exported + // functions, and we do not install anything. + // TODO(rsc): Once we know that caching is rebuilding + // at the right times (not missing rebuilds), here we should + // probably delete the installed header, if any. + if cfg.BuildX { + sh.ShowCmd("", "# %s not created", src) + } + return nil + } + + if err := AllowInstall(a); err != nil { + return err + } + + dir, _ := filepath.Split(a.Target) + if dir != "" { + if err := sh.Mkdir(dir); err != nil { + return err + } + } + + return sh.moveOrCopyFile(a.Target, src, 0666, true) +} + +// cover runs, in effect, +// +// go tool cover -mode=b.coverMode -var="varName" -o dst.go src.go +func (b *Builder) cover(a *Action, dst, src string, varName string) error { + return b.Shell(a).run(a.Objdir, "", nil, + cfg.BuildToolexec, + base.Tool("cover"), + "-mode", a.Package.Internal.Cover.Mode, + "-var", varName, + "-o", dst, + src) +} + +// cover2 runs, in effect, +// +// go tool cover -pkgcfg=<config file> -mode=b.coverMode -var="varName" -o <outfiles> <infiles> +// +// Return value is an updated output files list; in addition to the +// regular outputs (instrumented source files) the cover tool also +// writes a separate file (appearing first in the list of outputs) +// that will contain coverage counters and meta-data. +func (b *Builder) cover2(a *Action, infiles, outfiles []string, varName string, mode string) ([]string, error) { + pkgcfg := a.Objdir + "pkgcfg.txt" + covoutputs := a.Objdir + "coveroutfiles.txt" + odir := filepath.Dir(outfiles[0]) + cv := filepath.Join(odir, "covervars.go") + outfiles = append([]string{cv}, outfiles...) + if err := b.writeCoverPkgInputs(a, pkgcfg, covoutputs, outfiles); err != nil { + return nil, err + } + args := []string{base.Tool("cover"), + "-pkgcfg", pkgcfg, + "-mode", mode, + "-var", varName, + "-outfilelist", covoutputs, + } + args = append(args, infiles...) + if err := b.Shell(a).run(a.Objdir, "", nil, + cfg.BuildToolexec, args); err != nil { + return nil, err + } + return outfiles, nil +} + +func (b *Builder) writeCoverPkgInputs(a *Action, pconfigfile string, covoutputsfile string, outfiles []string) error { + sh := b.Shell(a) + p := a.Package + p.Internal.Cover.Cfg = a.Objdir + "coveragecfg" + pcfg := covcmd.CoverPkgConfig{ + PkgPath: p.ImportPath, + PkgName: p.Name, + // Note: coverage granularity is currently hard-wired to + // 'perblock'; there isn't a way using "go build -cover" or "go + // test -cover" to select it. This may change in the future + // depending on user demand. + Granularity: "perblock", + OutConfig: p.Internal.Cover.Cfg, + Local: p.Internal.Local, + } + if ba, ok := a.Actor.(*buildActor); ok && ba.covMetaFileName != "" { + pcfg.EmitMetaFile = a.Objdir + ba.covMetaFileName + } + if a.Package.Module != nil { + pcfg.ModulePath = a.Package.Module.Path + } + data, err := json.Marshal(pcfg) + if err != nil { + return err + } + data = append(data, '\n') + if err := sh.writeFile(pconfigfile, data); err != nil { + return err + } + var sb strings.Builder + for i := range outfiles { + fmt.Fprintf(&sb, "%s\n", outfiles[i]) + } + return sh.writeFile(covoutputsfile, []byte(sb.String())) +} + +var objectMagic = [][]byte{ + {'!', '<', 'a', 'r', 'c', 'h', '>', '\n'}, // Package archive + {'<', 'b', 'i', 'g', 'a', 'f', '>', '\n'}, // Package AIX big archive + {'\x7F', 'E', 'L', 'F'}, // ELF + {0xFE, 0xED, 0xFA, 0xCE}, // Mach-O big-endian 32-bit + {0xFE, 0xED, 0xFA, 0xCF}, // Mach-O big-endian 64-bit + {0xCE, 0xFA, 0xED, 0xFE}, // Mach-O little-endian 32-bit + {0xCF, 0xFA, 0xED, 0xFE}, // Mach-O little-endian 64-bit + {0x4d, 0x5a, 0x90, 0x00, 0x03, 0x00}, // PE (Windows) as generated by 6l/8l and gcc + {0x4d, 0x5a, 0x78, 0x00, 0x01, 0x00}, // PE (Windows) as generated by llvm for dll + {0x00, 0x00, 0x01, 0xEB}, // Plan 9 i386 + {0x00, 0x00, 0x8a, 0x97}, // Plan 9 amd64 + {0x00, 0x00, 0x06, 0x47}, // Plan 9 arm + {0x00, 0x61, 0x73, 0x6D}, // WASM + {0x01, 0xDF}, // XCOFF 32bit + {0x01, 0xF7}, // XCOFF 64bit +} + +func isObject(s string) bool { + f, err := os.Open(s) + if err != nil { + return false + } + defer f.Close() + buf := make([]byte, 64) + io.ReadFull(f, buf) + for _, magic := range objectMagic { + if bytes.HasPrefix(buf, magic) { + return true + } + } + return false +} + +// cCompilerEnv returns environment variables to set when running the +// C compiler. This is needed to disable escape codes in clang error +// messages that confuse tools like cgo. +func (b *Builder) cCompilerEnv() []string { + return []string{"TERM=dumb"} +} + +// mkAbs returns an absolute path corresponding to +// evaluating f in the directory dir. +// We always pass absolute paths of source files so that +// the error messages will include the full path to a file +// in need of attention. +func mkAbs(dir, f string) string { + // Leave absolute paths alone. + // Also, during -n mode we use the pseudo-directory $WORK + // instead of creating an actual work directory that won't be used. + // Leave paths beginning with $WORK alone too. + if filepath.IsAbs(f) || strings.HasPrefix(f, "$WORK") { + return f + } + return filepath.Join(dir, f) +} + +type toolchain interface { + // gc runs the compiler in a specific directory on a set of files + // and returns the name of the generated output file. + gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, out []byte, err error) + // cc runs the toolchain's C compiler in a directory on a C file + // to produce an output file. + cc(b *Builder, a *Action, ofile, cfile string) error + // asm runs the assembler in a specific directory on specific files + // and returns a list of named output files. + asm(b *Builder, a *Action, sfiles []string) ([]string, error) + // symabis scans the symbol ABIs from sfiles and returns the + // path to the output symbol ABIs file, or "" if none. + symabis(b *Builder, a *Action, sfiles []string) (string, error) + // pack runs the archive packer in a specific directory to create + // an archive from a set of object files. + // typically it is run in the object directory. + pack(b *Builder, a *Action, afile string, ofiles []string) error + // ld runs the linker to create an executable starting at mainpkg. + ld(b *Builder, root *Action, targetPath, importcfg, mainpkg string) error + // ldShared runs the linker to create a shared library containing the pkgs built by toplevelactions + ldShared(b *Builder, root *Action, toplevelactions []*Action, targetPath, importcfg string, allactions []*Action) error + + compiler() string + linker() string +} + +type noToolchain struct{} + +func noCompiler() error { + log.Fatalf("unknown compiler %q", cfg.BuildContext.Compiler) + return nil +} + +func (noToolchain) compiler() string { + noCompiler() + return "" +} + +func (noToolchain) linker() string { + noCompiler() + return "" +} + +func (noToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, out []byte, err error) { + return "", nil, noCompiler() +} + +func (noToolchain) asm(b *Builder, a *Action, sfiles []string) ([]string, error) { + return nil, noCompiler() +} + +func (noToolchain) symabis(b *Builder, a *Action, sfiles []string) (string, error) { + return "", noCompiler() +} + +func (noToolchain) pack(b *Builder, a *Action, afile string, ofiles []string) error { + return noCompiler() +} + +func (noToolchain) ld(b *Builder, root *Action, targetPath, importcfg, mainpkg string) error { + return noCompiler() +} + +func (noToolchain) ldShared(b *Builder, root *Action, toplevelactions []*Action, targetPath, importcfg string, allactions []*Action) error { + return noCompiler() +} + +func (noToolchain) cc(b *Builder, a *Action, ofile, cfile string) error { + return noCompiler() +} + +// gcc runs the gcc C compiler to create an object from a single C file. +func (b *Builder) gcc(a *Action, workdir, out string, flags []string, cfile string) error { + p := a.Package + return b.ccompile(a, out, flags, cfile, b.GccCmd(p.Dir, workdir)) +} + +// gxx runs the g++ C++ compiler to create an object from a single C++ file. +func (b *Builder) gxx(a *Action, workdir, out string, flags []string, cxxfile string) error { + p := a.Package + return b.ccompile(a, out, flags, cxxfile, b.GxxCmd(p.Dir, workdir)) +} + +// gfortran runs the gfortran Fortran compiler to create an object from a single Fortran file. +func (b *Builder) gfortran(a *Action, workdir, out string, flags []string, ffile string) error { + p := a.Package + return b.ccompile(a, out, flags, ffile, b.gfortranCmd(p.Dir, workdir)) +} + +// ccompile runs the given C or C++ compiler and creates an object from a single source file. +func (b *Builder) ccompile(a *Action, outfile string, flags []string, file string, compiler []string) error { + p := a.Package + sh := b.Shell(a) + file = mkAbs(p.Dir, file) + outfile = mkAbs(p.Dir, outfile) + + // Elide source directory paths if -trimpath or GOROOT_FINAL is set. + // This is needed for source files (e.g., a .c file in a package directory). + // TODO(golang.org/issue/36072): cgo also generates files with #line + // directives pointing to the source directory. It should not generate those + // when -trimpath is enabled. + if b.gccSupportsFlag(compiler, "-fdebug-prefix-map=a=b") { + if cfg.BuildTrimpath || p.Goroot { + prefixMapFlag := "-fdebug-prefix-map" + if b.gccSupportsFlag(compiler, "-ffile-prefix-map=a=b") { + prefixMapFlag = "-ffile-prefix-map" + } + // Keep in sync with Action.trimpath. + // The trimmed paths are a little different, but we need to trim in mostly the + // same situations. + var from, toPath string + if m := p.Module; m == nil { + if p.Root == "" { // command-line-arguments in GOPATH mode, maybe? + from = p.Dir + toPath = p.ImportPath + } else if p.Goroot { + from = p.Root + toPath = "GOROOT" + } else { + from = p.Root + toPath = "GOPATH" + } + } else if m.Dir == "" { + // The module is in the vendor directory. Replace the entire vendor + // directory path, because the module's Dir is not filled in. + from = modload.VendorDir() + toPath = "vendor" + } else { + from = m.Dir + toPath = m.Path + if m.Version != "" { + toPath += "@" + m.Version + } + } + // -fdebug-prefix-map (or -ffile-prefix-map) requires an absolute "to" + // path (or it joins the path with the working directory). Pick something + // that makes sense for the target platform. + var to string + if cfg.BuildContext.GOOS == "windows" { + to = filepath.Join(`\\_\_`, toPath) + } else { + to = filepath.Join("/_", toPath) + } + flags = append(slices.Clip(flags), prefixMapFlag+"="+from+"="+to) + } + } + + // Tell gcc to not insert truly random numbers into the build process + // this ensures LTO won't create random numbers for symbols. + if b.gccSupportsFlag(compiler, "-frandom-seed=1") { + flags = append(flags, "-frandom-seed="+buildid.HashToString(a.actionID)) + } + + overlayPath := file + if p, ok := a.nonGoOverlay[overlayPath]; ok { + overlayPath = p + } + output, err := sh.runOut(filepath.Dir(overlayPath), b.cCompilerEnv(), compiler, flags, "-o", outfile, "-c", filepath.Base(overlayPath)) + + // On FreeBSD 11, when we pass -g to clang 3.8 it + // invokes its internal assembler with -dwarf-version=2. + // When it sees .section .note.GNU-stack, it warns + // "DWARF2 only supports one section per compilation unit". + // This warning makes no sense, since the section is empty, + // but it confuses people. + // We work around the problem by detecting the warning + // and dropping -g and trying again. + if bytes.Contains(output, []byte("DWARF2 only supports one section per compilation unit")) { + newFlags := make([]string, 0, len(flags)) + for _, f := range flags { + if !strings.HasPrefix(f, "-g") { + newFlags = append(newFlags, f) + } + } + if len(newFlags) < len(flags) { + return b.ccompile(a, outfile, newFlags, file, compiler) + } + } + + if len(output) > 0 && err == nil && os.Getenv("GO_BUILDER_NAME") != "" { + output = append(output, "C compiler warning promoted to error on Go builders\n"...) + err = errors.New("warning promoted to error") + } + + return sh.reportCmd("", "", output, err) +} + +// gccld runs the gcc linker to create an executable from a set of object files. +// Any error output is only displayed for BuildN or BuildX. +func (b *Builder) gccld(a *Action, objdir, outfile string, flags []string, objs []string) error { + p := a.Package + sh := b.Shell(a) + var cmd []string + if len(p.CXXFiles) > 0 || len(p.SwigCXXFiles) > 0 { + cmd = b.GxxCmd(p.Dir, objdir) + } else { + cmd = b.GccCmd(p.Dir, objdir) + } + + cmdargs := []any{cmd, "-o", outfile, objs, flags} + out, err := sh.runOut(base.Cwd(), b.cCompilerEnv(), cmdargs...) + + if len(out) > 0 { + // Filter out useless linker warnings caused by bugs outside Go. + // See also cmd/link/internal/ld's hostlink method. + var save [][]byte + var skipLines int + for _, line := range bytes.SplitAfter(out, []byte("\n")) { + // golang.org/issue/26073 - Apple Xcode bug + if bytes.Contains(line, []byte("ld: warning: text-based stub file")) { + continue + } + + if skipLines > 0 { + skipLines-- + continue + } + + // Remove duplicate main symbol with runtime/cgo on AIX. + // With runtime/cgo, two main are available: + // One is generated by cgo tool with {return 0;}. + // The other one is the main calling runtime.rt0_go + // in runtime/cgo. + // The second can't be used by cgo programs because + // runtime.rt0_go is unknown to them. + // Therefore, we let ld remove this main version + // and used the cgo generated one. + if p.ImportPath == "runtime/cgo" && bytes.Contains(line, []byte("ld: 0711-224 WARNING: Duplicate symbol: .main")) { + skipLines = 1 + continue + } + + save = append(save, line) + } + out = bytes.Join(save, nil) + } + // Note that failure is an expected outcome here, so we report output only + // in debug mode and don't report the error. + if cfg.BuildN || cfg.BuildX { + sh.reportCmd("", "", out, nil) + } + return err +} + +// GccCmd returns a gcc command line prefix +// defaultCC is defined in zdefaultcc.go, written by cmd/dist. +func (b *Builder) GccCmd(incdir, workdir string) []string { + return b.compilerCmd(b.ccExe(), incdir, workdir) +} + +// GxxCmd returns a g++ command line prefix +// defaultCXX is defined in zdefaultcc.go, written by cmd/dist. +func (b *Builder) GxxCmd(incdir, workdir string) []string { + return b.compilerCmd(b.cxxExe(), incdir, workdir) +} + +// gfortranCmd returns a gfortran command line prefix. +func (b *Builder) gfortranCmd(incdir, workdir string) []string { + return b.compilerCmd(b.fcExe(), incdir, workdir) +} + +// ccExe returns the CC compiler setting without all the extra flags we add implicitly. +func (b *Builder) ccExe() []string { + return envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch)) +} + +// cxxExe returns the CXX compiler setting without all the extra flags we add implicitly. +func (b *Builder) cxxExe() []string { + return envList("CXX", cfg.DefaultCXX(cfg.Goos, cfg.Goarch)) +} + +// fcExe returns the FC compiler setting without all the extra flags we add implicitly. +func (b *Builder) fcExe() []string { + return envList("FC", "gfortran") +} + +// compilerCmd returns a command line prefix for the given environment +// variable and using the default command when the variable is empty. +func (b *Builder) compilerCmd(compiler []string, incdir, workdir string) []string { + a := append(compiler, "-I", incdir) + + // Definitely want -fPIC but on Windows gcc complains + // "-fPIC ignored for target (all code is position independent)" + if cfg.Goos != "windows" { + a = append(a, "-fPIC") + } + a = append(a, b.gccArchArgs()...) + // gcc-4.5 and beyond require explicit "-pthread" flag + // for multithreading with pthread library. + if cfg.BuildContext.CgoEnabled { + switch cfg.Goos { + case "windows": + a = append(a, "-mthreads") + default: + a = append(a, "-pthread") + } + } + + if cfg.Goos == "aix" { + // mcmodel=large must always be enabled to allow large TOC. + a = append(a, "-mcmodel=large") + } + + // disable ASCII art in clang errors, if possible + if b.gccSupportsFlag(compiler, "-fno-caret-diagnostics") { + a = append(a, "-fno-caret-diagnostics") + } + // clang is too smart about command-line arguments + if b.gccSupportsFlag(compiler, "-Qunused-arguments") { + a = append(a, "-Qunused-arguments") + } + + // zig cc passes --gc-sections to the underlying linker, which then causes + // undefined symbol errors when compiling with cgo but without C code. + // https://github.com/golang/go/issues/52690 + if b.gccSupportsFlag(compiler, "-Wl,--no-gc-sections") { + a = append(a, "-Wl,--no-gc-sections") + } + + // disable word wrapping in error messages + a = append(a, "-fmessage-length=0") + + // Tell gcc not to include the work directory in object files. + if b.gccSupportsFlag(compiler, "-fdebug-prefix-map=a=b") { + if workdir == "" { + workdir = b.WorkDir + } + workdir = strings.TrimSuffix(workdir, string(filepath.Separator)) + if b.gccSupportsFlag(compiler, "-ffile-prefix-map=a=b") { + a = append(a, "-ffile-prefix-map="+workdir+"=/tmp/go-build") + } else { + a = append(a, "-fdebug-prefix-map="+workdir+"=/tmp/go-build") + } + } + + // Tell gcc not to include flags in object files, which defeats the + // point of -fdebug-prefix-map above. + if b.gccSupportsFlag(compiler, "-gno-record-gcc-switches") { + a = append(a, "-gno-record-gcc-switches") + } + + // On OS X, some of the compilers behave as if -fno-common + // is always set, and the Mach-O linker in 6l/8l assumes this. + // See https://golang.org/issue/3253. + if cfg.Goos == "darwin" || cfg.Goos == "ios" { + a = append(a, "-fno-common") + } + + return a +} + +// gccNoPie returns the flag to use to request non-PIE. On systems +// with PIE (position independent executables) enabled by default, +// -no-pie must be passed when doing a partial link with -Wl,-r. +// But -no-pie is not supported by all compilers, and clang spells it -nopie. +func (b *Builder) gccNoPie(linker []string) string { + if b.gccSupportsFlag(linker, "-no-pie") { + return "-no-pie" + } + if b.gccSupportsFlag(linker, "-nopie") { + return "-nopie" + } + return "" +} + +// gccSupportsFlag checks to see if the compiler supports a flag. +func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool { + // We use the background shell for operations here because, while this is + // triggered by some Action, it's not really about that Action, and often we + // just get the results from the global cache. + sh := b.BackgroundShell() + + key := [2]string{compiler[0], flag} + + // We used to write an empty C file, but that gets complicated with go + // build -n. We tried using a file that does not exist, but that fails on + // systems with GCC version 4.2.1; that is the last GPLv2 version of GCC, + // so some systems have frozen on it. Now we pass an empty file on stdin, + // which should work at least for GCC and clang. + // + // If the argument is "-Wl,", then it is testing the linker. In that case, + // skip "-c". If it's not "-Wl,", then we are testing the compiler and can + // omit the linking step with "-c". + // + // Using the same CFLAGS/LDFLAGS here and for building the program. + + // On the iOS builder the command + // $CC -Wl,--no-gc-sections -x c - -o /dev/null < /dev/null + // is failing with: + // Unable to remove existing file: Invalid argument + tmp := os.DevNull + if runtime.GOOS == "windows" || runtime.GOOS == "ios" { + f, err := os.CreateTemp(b.WorkDir, "") + if err != nil { + return false + } + f.Close() + tmp = f.Name() + defer os.Remove(tmp) + } + + cmdArgs := str.StringList(compiler, flag) + if strings.HasPrefix(flag, "-Wl,") /* linker flag */ { + ldflags, err := buildFlags("LDFLAGS", defaultCFlags, nil, checkLinkerFlags) + if err != nil { + return false + } + cmdArgs = append(cmdArgs, ldflags...) + } else { /* compiler flag, add "-c" */ + cflags, err := buildFlags("CFLAGS", defaultCFlags, nil, checkCompilerFlags) + if err != nil { + return false + } + cmdArgs = append(cmdArgs, cflags...) + cmdArgs = append(cmdArgs, "-c") + } + + cmdArgs = append(cmdArgs, "-x", "c", "-", "-o", tmp) + + if cfg.BuildN { + sh.ShowCmd(b.WorkDir, "%s || true", joinUnambiguously(cmdArgs)) + return false + } + + // gccCompilerID acquires b.exec, so do before acquiring lock. + compilerID, cacheOK := b.gccCompilerID(compiler[0]) + + b.exec.Lock() + defer b.exec.Unlock() + if b, ok := b.flagCache[key]; ok { + return b + } + if b.flagCache == nil { + b.flagCache = make(map[[2]string]bool) + } + + // Look in build cache. + var flagID cache.ActionID + if cacheOK { + flagID = cache.Subkey(compilerID, "gccSupportsFlag "+flag) + if data, _, err := cache.GetBytes(cache.Default(), flagID); err == nil { + supported := string(data) == "true" + b.flagCache[key] = supported + return supported + } + } + + if cfg.BuildX { + sh.ShowCmd(b.WorkDir, "%s || true", joinUnambiguously(cmdArgs)) + } + cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...) + cmd.Dir = b.WorkDir + cmd.Env = append(cmd.Environ(), "LC_ALL=C") + out, _ := cmd.CombinedOutput() + // GCC says "unrecognized command line option". + // clang says "unknown argument". + // tcc says "unsupported" + // AIX says "not recognized" + // Older versions of GCC say "unrecognised debug output level". + // For -fsplit-stack GCC says "'-fsplit-stack' is not supported". + supported := !bytes.Contains(out, []byte("unrecognized")) && + !bytes.Contains(out, []byte("unknown")) && + !bytes.Contains(out, []byte("unrecognised")) && + !bytes.Contains(out, []byte("is not supported")) && + !bytes.Contains(out, []byte("not recognized")) && + !bytes.Contains(out, []byte("unsupported")) + + if cacheOK { + s := "false" + if supported { + s = "true" + } + cache.PutBytes(cache.Default(), flagID, []byte(s)) + } + + b.flagCache[key] = supported + return supported +} + +// statString returns a string form of an os.FileInfo, for serializing and comparison. +func statString(info os.FileInfo) string { + return fmt.Sprintf("stat %d %x %v %v\n", info.Size(), uint64(info.Mode()), info.ModTime(), info.IsDir()) +} + +// gccCompilerID returns a build cache key for the current gcc, +// as identified by running 'compiler'. +// The caller can use subkeys of the key. +// Other parts of cmd/go can use the id as a hash +// of the installed compiler version. +func (b *Builder) gccCompilerID(compiler string) (id cache.ActionID, ok bool) { + // We use the background shell for operations here because, while this is + // triggered by some Action, it's not really about that Action, and often we + // just get the results from the global cache. + sh := b.BackgroundShell() + + if cfg.BuildN { + sh.ShowCmd(b.WorkDir, "%s || true", joinUnambiguously([]string{compiler, "--version"})) + return cache.ActionID{}, false + } + + b.exec.Lock() + defer b.exec.Unlock() + + if id, ok := b.gccCompilerIDCache[compiler]; ok { + return id, ok + } + + // We hash the compiler's full path to get a cache entry key. + // That cache entry holds a validation description, + // which is of the form: + // + // filename \x00 statinfo \x00 + // ... + // compiler id + // + // If os.Stat of each filename matches statinfo, + // then the entry is still valid, and we can use the + // compiler id without any further expense. + // + // Otherwise, we compute a new validation description + // and compiler id (below). + exe, err := cfg.LookPath(compiler) + if err != nil { + return cache.ActionID{}, false + } + + h := cache.NewHash("gccCompilerID") + fmt.Fprintf(h, "gccCompilerID %q", exe) + key := h.Sum() + data, _, err := cache.GetBytes(cache.Default(), key) + if err == nil && len(data) > len(id) { + stats := strings.Split(string(data[:len(data)-len(id)]), "\x00") + if len(stats)%2 != 0 { + goto Miss + } + for i := 0; i+2 <= len(stats); i++ { + info, err := os.Stat(stats[i]) + if err != nil || statString(info) != stats[i+1] { + goto Miss + } + } + copy(id[:], data[len(data)-len(id):]) + return id, true + Miss: + } + + // Validation failed. Compute a new description (in buf) and compiler ID (in h). + // For now, there are only at most two filenames in the stat information. + // The first one is the compiler executable we invoke. + // The second is the underlying compiler as reported by -v -### + // (see b.gccToolID implementation in buildid.go). + toolID, exe2, err := b.gccToolID(compiler, "c") + if err != nil { + return cache.ActionID{}, false + } + + exes := []string{exe, exe2} + str.Uniq(&exes) + fmt.Fprintf(h, "gccCompilerID %q %q\n", exes, toolID) + id = h.Sum() + + var buf bytes.Buffer + for _, exe := range exes { + if exe == "" { + continue + } + info, err := os.Stat(exe) + if err != nil { + return cache.ActionID{}, false + } + buf.WriteString(exe) + buf.WriteString("\x00") + buf.WriteString(statString(info)) + buf.WriteString("\x00") + } + buf.Write(id[:]) + + cache.PutBytes(cache.Default(), key, buf.Bytes()) + if b.gccCompilerIDCache == nil { + b.gccCompilerIDCache = make(map[string]cache.ActionID) + } + b.gccCompilerIDCache[compiler] = id + return id, true +} + +// gccArchArgs returns arguments to pass to gcc based on the architecture. +func (b *Builder) gccArchArgs() []string { + switch cfg.Goarch { + case "386": + return []string{"-m32"} + case "amd64": + if cfg.Goos == "darwin" { + return []string{"-arch", "x86_64", "-m64"} + } + return []string{"-m64"} + case "arm64": + if cfg.Goos == "darwin" { + return []string{"-arch", "arm64"} + } + case "arm": + return []string{"-marm"} // not thumb + case "s390x": + return []string{"-m64", "-march=z196"} + case "mips64", "mips64le": + args := []string{"-mabi=64"} + if cfg.GOMIPS64 == "hardfloat" { + return append(args, "-mhard-float") + } else if cfg.GOMIPS64 == "softfloat" { + return append(args, "-msoft-float") + } + case "mips", "mipsle": + args := []string{"-mabi=32", "-march=mips32"} + if cfg.GOMIPS == "hardfloat" { + return append(args, "-mhard-float", "-mfp32", "-mno-odd-spreg") + } else if cfg.GOMIPS == "softfloat" { + return append(args, "-msoft-float") + } + case "loong64": + return []string{"-mabi=lp64d"} + case "ppc64": + if cfg.Goos == "aix" { + return []string{"-maix64"} + } + } + return nil +} + +// envList returns the value of the given environment variable broken +// into fields, using the default value when the variable is empty. +// +// The environment variable must be quoted correctly for +// quoted.Split. This should be done before building +// anything, for example, in BuildInit. +func envList(key, def string) []string { + v := cfg.Getenv(key) + if v == "" { + v = def + } + args, err := quoted.Split(v) + if err != nil { + panic(fmt.Sprintf("could not parse environment variable %s with value %q: %v", key, v, err)) + } + return args +} + +// CFlags returns the flags to use when invoking the C, C++ or Fortran compilers, or cgo. +func (b *Builder) CFlags(p *load.Package) (cppflags, cflags, cxxflags, fflags, ldflags []string, err error) { + if cppflags, err = buildFlags("CPPFLAGS", "", p.CgoCPPFLAGS, checkCompilerFlags); err != nil { + return + } + if cflags, err = buildFlags("CFLAGS", defaultCFlags, p.CgoCFLAGS, checkCompilerFlags); err != nil { + return + } + if cxxflags, err = buildFlags("CXXFLAGS", defaultCFlags, p.CgoCXXFLAGS, checkCompilerFlags); err != nil { + return + } + if fflags, err = buildFlags("FFLAGS", defaultCFlags, p.CgoFFLAGS, checkCompilerFlags); err != nil { + return + } + if ldflags, err = buildFlags("LDFLAGS", defaultCFlags, p.CgoLDFLAGS, checkLinkerFlags); err != nil { + return + } + + return +} + +func buildFlags(name, defaults string, fromPackage []string, check func(string, string, []string) error) ([]string, error) { + if err := check(name, "#cgo "+name, fromPackage); err != nil { + return nil, err + } + return str.StringList(envList("CGO_"+name, defaults), fromPackage), nil +} + +var cgoRe = lazyregexp.New(`[/\\:]`) + +func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgofiles, gccfiles, gxxfiles, mfiles, ffiles []string) (outGo, outObj []string, err error) { + p := a.Package + sh := b.Shell(a) + + cgoCPPFLAGS, cgoCFLAGS, cgoCXXFLAGS, cgoFFLAGS, cgoLDFLAGS, err := b.CFlags(p) + if err != nil { + return nil, nil, err + } + + cgoCPPFLAGS = append(cgoCPPFLAGS, pcCFLAGS...) + cgoLDFLAGS = append(cgoLDFLAGS, pcLDFLAGS...) + // If we are compiling Objective-C code, then we need to link against libobjc + if len(mfiles) > 0 { + cgoLDFLAGS = append(cgoLDFLAGS, "-lobjc") + } + + // Likewise for Fortran, except there are many Fortran compilers. + // Support gfortran out of the box and let others pass the correct link options + // via CGO_LDFLAGS + if len(ffiles) > 0 { + fc := cfg.Getenv("FC") + if fc == "" { + fc = "gfortran" + } + if strings.Contains(fc, "gfortran") { + cgoLDFLAGS = append(cgoLDFLAGS, "-lgfortran") + } + } + + // Scrutinize CFLAGS and related for flags that might cause + // problems if we are using internal linking (for example, use of + // plugins, LTO, etc) by calling a helper routine that builds on + // the existing CGO flags allow-lists. If we see anything + // suspicious, emit a special token file "preferlinkext" (known to + // the linker) in the object file to signal the that it should not + // try to link internally and should revert to external linking. + // The token we pass is a suggestion, not a mandate; if a user is + // explicitly asking for a specific linkmode via the "-linkmode" + // flag, the token will be ignored. NB: in theory we could ditch + // the token approach and just pass a flag to the linker when we + // eventually invoke it, and the linker flag could then be + // documented (although coming up with a simple explanation of the + // flag might be challenging). For more context see issues #58619, + // #58620, and #58848. + flagSources := []string{"CGO_CFLAGS", "CGO_CXXFLAGS", "CGO_FFLAGS"} + flagLists := [][]string{cgoCFLAGS, cgoCXXFLAGS, cgoFFLAGS} + if flagsNotCompatibleWithInternalLinking(flagSources, flagLists) { + tokenFile := objdir + "preferlinkext" + if err := sh.writeFile(tokenFile, nil); err != nil { + return nil, nil, err + } + outObj = append(outObj, tokenFile) + } + + if cfg.BuildMSan { + cgoCFLAGS = append([]string{"-fsanitize=memory"}, cgoCFLAGS...) + cgoLDFLAGS = append([]string{"-fsanitize=memory"}, cgoLDFLAGS...) + } + if cfg.BuildASan { + cgoCFLAGS = append([]string{"-fsanitize=address"}, cgoCFLAGS...) + cgoLDFLAGS = append([]string{"-fsanitize=address"}, cgoLDFLAGS...) + } + + // Allows including _cgo_export.h, as well as the user's .h files, + // from .[ch] files in the package. + cgoCPPFLAGS = append(cgoCPPFLAGS, "-I", objdir) + + // cgo + // TODO: CGO_FLAGS? + gofiles := []string{objdir + "_cgo_gotypes.go"} + cfiles := []string{"_cgo_export.c"} + for _, fn := range cgofiles { + f := strings.TrimSuffix(filepath.Base(fn), ".go") + gofiles = append(gofiles, objdir+f+".cgo1.go") + cfiles = append(cfiles, f+".cgo2.c") + } + + // TODO: make cgo not depend on $GOARCH? + + cgoflags := []string{} + if p.Standard && p.ImportPath == "runtime/cgo" { + cgoflags = append(cgoflags, "-import_runtime_cgo=false") + } + if p.Standard && (p.ImportPath == "runtime/race" || p.ImportPath == "runtime/msan" || p.ImportPath == "runtime/cgo" || p.ImportPath == "runtime/asan") { + cgoflags = append(cgoflags, "-import_syscall=false") + } + + // Update $CGO_LDFLAGS with p.CgoLDFLAGS. + // These flags are recorded in the generated _cgo_gotypes.go file + // using //go:cgo_ldflag directives, the compiler records them in the + // object file for the package, and then the Go linker passes them + // along to the host linker. At this point in the code, cgoLDFLAGS + // consists of the original $CGO_LDFLAGS (unchecked) and all the + // flags put together from source code (checked). + cgoenv := b.cCompilerEnv() + if len(cgoLDFLAGS) > 0 { + flags := make([]string, len(cgoLDFLAGS)) + for i, f := range cgoLDFLAGS { + flags[i] = strconv.Quote(f) + } + cgoenv = append(cgoenv, "CGO_LDFLAGS="+strings.Join(flags, " ")) + } + + if cfg.BuildToolchainName == "gccgo" { + if b.gccSupportsFlag([]string{BuildToolchain.compiler()}, "-fsplit-stack") { + cgoCFLAGS = append(cgoCFLAGS, "-fsplit-stack") + } + cgoflags = append(cgoflags, "-gccgo") + if pkgpath := gccgoPkgpath(p); pkgpath != "" { + cgoflags = append(cgoflags, "-gccgopkgpath="+pkgpath) + } + if !BuildToolchain.(gccgoToolchain).supportsCgoIncomplete(b, a) { + cgoflags = append(cgoflags, "-gccgo_define_cgoincomplete") + } + } + + switch cfg.BuildBuildmode { + case "c-archive", "c-shared": + // Tell cgo that if there are any exported functions + // it should generate a header file that C code can + // #include. + cgoflags = append(cgoflags, "-exportheader="+objdir+"_cgo_install.h") + } + + // Rewrite overlaid paths in cgo files. + // cgo adds //line and #line pragmas in generated files with these paths. + var trimpath []string + for i := range cgofiles { + path := mkAbs(p.Dir, cgofiles[i]) + if opath, ok := fsys.OverlayPath(path); ok { + cgofiles[i] = opath + trimpath = append(trimpath, opath+"=>"+path) + } + } + if len(trimpath) > 0 { + cgoflags = append(cgoflags, "-trimpath", strings.Join(trimpath, ";")) + } + + if err := sh.run(p.Dir, p.ImportPath, cgoenv, cfg.BuildToolexec, cgoExe, "-objdir", objdir, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil { + return nil, nil, err + } + outGo = append(outGo, gofiles...) + + // Use sequential object file names to keep them distinct + // and short enough to fit in the .a header file name slots. + // We no longer collect them all into _all.o, and we'd like + // tools to see both the .o suffix and unique names, so + // we need to make them short enough not to be truncated + // in the final archive. + oseq := 0 + nextOfile := func() string { + oseq++ + return objdir + fmt.Sprintf("_x%03d.o", oseq) + } + + // gcc + cflags := str.StringList(cgoCPPFLAGS, cgoCFLAGS) + for _, cfile := range cfiles { + ofile := nextOfile() + if err := b.gcc(a, a.Objdir, ofile, cflags, objdir+cfile); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + for _, file := range gccfiles { + ofile := nextOfile() + if err := b.gcc(a, a.Objdir, ofile, cflags, file); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + cxxflags := str.StringList(cgoCPPFLAGS, cgoCXXFLAGS) + for _, file := range gxxfiles { + ofile := nextOfile() + if err := b.gxx(a, a.Objdir, ofile, cxxflags, file); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + for _, file := range mfiles { + ofile := nextOfile() + if err := b.gcc(a, a.Objdir, ofile, cflags, file); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + fflags := str.StringList(cgoCPPFLAGS, cgoFFLAGS) + for _, file := range ffiles { + ofile := nextOfile() + if err := b.gfortran(a, a.Objdir, ofile, fflags, file); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + switch cfg.BuildToolchainName { + case "gc": + importGo := objdir + "_cgo_import.go" + dynOutGo, dynOutObj, err := b.dynimport(a, objdir, importGo, cgoExe, cflags, cgoLDFLAGS, outObj) + if err != nil { + return nil, nil, err + } + if dynOutGo != "" { + outGo = append(outGo, dynOutGo) + } + if dynOutObj != "" { + outObj = append(outObj, dynOutObj) + } + + case "gccgo": + defunC := objdir + "_cgo_defun.c" + defunObj := objdir + "_cgo_defun.o" + if err := BuildToolchain.cc(b, a, defunObj, defunC); err != nil { + return nil, nil, err + } + outObj = append(outObj, defunObj) + + default: + noCompiler() + } + + // Double check the //go:cgo_ldflag comments in the generated files. + // The compiler only permits such comments in files whose base name + // starts with "_cgo_". Make sure that the comments in those files + // are safe. This is a backstop against people somehow smuggling + // such a comment into a file generated by cgo. + if cfg.BuildToolchainName == "gc" && !cfg.BuildN { + var flags []string + for _, f := range outGo { + if !strings.HasPrefix(filepath.Base(f), "_cgo_") { + continue + } + + src, err := os.ReadFile(f) + if err != nil { + return nil, nil, err + } + + const cgoLdflag = "//go:cgo_ldflag" + idx := bytes.Index(src, []byte(cgoLdflag)) + for idx >= 0 { + // We are looking at //go:cgo_ldflag. + // Find start of line. + start := bytes.LastIndex(src[:idx], []byte("\n")) + if start == -1 { + start = 0 + } + + // Find end of line. + end := bytes.Index(src[idx:], []byte("\n")) + if end == -1 { + end = len(src) + } else { + end += idx + } + + // Check for first line comment in line. + // We don't worry about /* */ comments, + // which normally won't appear in files + // generated by cgo. + commentStart := bytes.Index(src[start:], []byte("//")) + commentStart += start + // If that line comment is //go:cgo_ldflag, + // it's a match. + if bytes.HasPrefix(src[commentStart:], []byte(cgoLdflag)) { + // Pull out the flag, and unquote it. + // This is what the compiler does. + flag := string(src[idx+len(cgoLdflag) : end]) + flag = strings.TrimSpace(flag) + flag = strings.Trim(flag, `"`) + flags = append(flags, flag) + } + src = src[end:] + idx = bytes.Index(src, []byte(cgoLdflag)) + } + } + + // We expect to find the contents of cgoLDFLAGS in flags. + if len(cgoLDFLAGS) > 0 { + outer: + for i := range flags { + for j, f := range cgoLDFLAGS { + if f != flags[i+j] { + continue outer + } + } + flags = append(flags[:i], flags[i+len(cgoLDFLAGS):]...) + break + } + } + + if err := checkLinkerFlags("LDFLAGS", "go:cgo_ldflag", flags); err != nil { + return nil, nil, err + } + } + + return outGo, outObj, nil +} + +// flagsNotCompatibleWithInternalLinking scans the list of cgo +// compiler flags (C/C++/Fortran) looking for flags that might cause +// problems if the build in question uses internal linking. The +// primary culprits are use of plugins or use of LTO, but we err on +// the side of caution, supporting only those flags that are on the +// allow-list for safe flags from security perspective. Return is TRUE +// if a sensitive flag is found, FALSE otherwise. +func flagsNotCompatibleWithInternalLinking(sourceList []string, flagListList [][]string) bool { + for i := range sourceList { + sn := sourceList[i] + fll := flagListList[i] + if err := checkCompilerFlagsForInternalLink(sn, sn, fll); err != nil { + return true + } + } + return false +} + +// dynimport creates a Go source file named importGo containing +// //go:cgo_import_dynamic directives for each symbol or library +// dynamically imported by the object files outObj. +// dynOutGo, if not empty, is a new Go file to build as part of the package. +// dynOutObj, if not empty, is a new file to add to the generated archive. +func (b *Builder) dynimport(a *Action, objdir, importGo, cgoExe string, cflags, cgoLDFLAGS, outObj []string) (dynOutGo, dynOutObj string, err error) { + p := a.Package + sh := b.Shell(a) + + cfile := objdir + "_cgo_main.c" + ofile := objdir + "_cgo_main.o" + if err := b.gcc(a, objdir, ofile, cflags, cfile); err != nil { + return "", "", err + } + + // Gather .syso files from this package and all (transitive) dependencies. + var syso []string + seen := make(map[*Action]bool) + var gatherSyso func(*Action) + gatherSyso = func(a1 *Action) { + if seen[a1] { + return + } + seen[a1] = true + if p1 := a1.Package; p1 != nil { + syso = append(syso, mkAbsFiles(p1.Dir, p1.SysoFiles)...) + } + for _, a2 := range a1.Deps { + gatherSyso(a2) + } + } + gatherSyso(a) + sort.Strings(syso) + str.Uniq(&syso) + linkobj := str.StringList(ofile, outObj, syso) + dynobj := objdir + "_cgo_.o" + + ldflags := cgoLDFLAGS + if (cfg.Goarch == "arm" && cfg.Goos == "linux") || cfg.Goos == "android" { + if !str.Contains(ldflags, "-no-pie") { + // we need to use -pie for Linux/ARM to get accurate imported sym (added in https://golang.org/cl/5989058) + // this seems to be outdated, but we don't want to break existing builds depending on this (Issue 45940) + ldflags = append(ldflags, "-pie") + } + if str.Contains(ldflags, "-pie") && str.Contains(ldflags, "-static") { + // -static -pie doesn't make sense, and causes link errors. + // Issue 26197. + n := make([]string, 0, len(ldflags)-1) + for _, flag := range ldflags { + if flag != "-static" { + n = append(n, flag) + } + } + ldflags = n + } + } + if err := b.gccld(a, objdir, dynobj, ldflags, linkobj); err != nil { + // We only need this information for internal linking. + // If this link fails, mark the object as requiring + // external linking. This link can fail for things like + // syso files that have unexpected dependencies. + // cmd/link explicitly looks for the name "dynimportfail". + // See issue #52863. + fail := objdir + "dynimportfail" + if err := sh.writeFile(fail, nil); err != nil { + return "", "", err + } + return "", fail, nil + } + + // cgo -dynimport + var cgoflags []string + if p.Standard && p.ImportPath == "runtime/cgo" { + cgoflags = []string{"-dynlinker"} // record path to dynamic linker + } + err = sh.run(base.Cwd(), p.ImportPath, b.cCompilerEnv(), cfg.BuildToolexec, cgoExe, "-dynpackage", p.Name, "-dynimport", dynobj, "-dynout", importGo, cgoflags) + if err != nil { + return "", "", err + } + return importGo, "", nil +} + +// Run SWIG on all SWIG input files. +// TODO: Don't build a shared library, once SWIG emits the necessary +// pragmas for external linking. +func (b *Builder) swig(a *Action, objdir string, pcCFLAGS []string) (outGo, outC, outCXX []string, err error) { + p := a.Package + + if err := b.swigVersionCheck(); err != nil { + return nil, nil, nil, err + } + + intgosize, err := b.swigIntSize(objdir) + if err != nil { + return nil, nil, nil, err + } + + for _, f := range p.SwigFiles { + goFile, cFile, err := b.swigOne(a, f, objdir, pcCFLAGS, false, intgosize) + if err != nil { + return nil, nil, nil, err + } + if goFile != "" { + outGo = append(outGo, goFile) + } + if cFile != "" { + outC = append(outC, cFile) + } + } + for _, f := range p.SwigCXXFiles { + goFile, cxxFile, err := b.swigOne(a, f, objdir, pcCFLAGS, true, intgosize) + if err != nil { + return nil, nil, nil, err + } + if goFile != "" { + outGo = append(outGo, goFile) + } + if cxxFile != "" { + outCXX = append(outCXX, cxxFile) + } + } + return outGo, outC, outCXX, nil +} + +// Make sure SWIG is new enough. +var ( + swigCheckOnce sync.Once + swigCheck error +) + +func (b *Builder) swigDoVersionCheck() error { + sh := b.BackgroundShell() + out, err := sh.runOut(".", nil, "swig", "-version") + if err != nil { + return err + } + re := regexp.MustCompile(`[vV]ersion +(\d+)([.]\d+)?([.]\d+)?`) + matches := re.FindSubmatch(out) + if matches == nil { + // Can't find version number; hope for the best. + return nil + } + + major, err := strconv.Atoi(string(matches[1])) + if err != nil { + // Can't find version number; hope for the best. + return nil + } + const errmsg = "must have SWIG version >= 3.0.6" + if major < 3 { + return errors.New(errmsg) + } + if major > 3 { + // 4.0 or later + return nil + } + + // We have SWIG version 3.x. + if len(matches[2]) > 0 { + minor, err := strconv.Atoi(string(matches[2][1:])) + if err != nil { + return nil + } + if minor > 0 { + // 3.1 or later + return nil + } + } + + // We have SWIG version 3.0.x. + if len(matches[3]) > 0 { + patch, err := strconv.Atoi(string(matches[3][1:])) + if err != nil { + return nil + } + if patch < 6 { + // Before 3.0.6. + return errors.New(errmsg) + } + } + + return nil +} + +func (b *Builder) swigVersionCheck() error { + swigCheckOnce.Do(func() { + swigCheck = b.swigDoVersionCheck() + }) + return swigCheck +} + +// Find the value to pass for the -intgosize option to swig. +var ( + swigIntSizeOnce sync.Once + swigIntSize string + swigIntSizeError error +) + +// This code fails to build if sizeof(int) <= 32 +const swigIntSizeCode = ` +package main +const i int = 1 << 32 +` + +// Determine the size of int on the target system for the -intgosize option +// of swig >= 2.0.9. Run only once. +func (b *Builder) swigDoIntSize(objdir string) (intsize string, err error) { + if cfg.BuildN { + return "$INTBITS", nil + } + src := filepath.Join(b.WorkDir, "swig_intsize.go") + if err = os.WriteFile(src, []byte(swigIntSizeCode), 0666); err != nil { + return + } + srcs := []string{src} + + p := load.GoFilesPackage(context.TODO(), load.PackageOpts{}, srcs) + + if _, _, e := BuildToolchain.gc(b, &Action{Mode: "swigDoIntSize", Package: p, Objdir: objdir}, "", nil, nil, "", false, srcs); e != nil { + return "32", nil + } + return "64", nil +} + +// Determine the size of int on the target system for the -intgosize option +// of swig >= 2.0.9. +func (b *Builder) swigIntSize(objdir string) (intsize string, err error) { + swigIntSizeOnce.Do(func() { + swigIntSize, swigIntSizeError = b.swigDoIntSize(objdir) + }) + return swigIntSize, swigIntSizeError +} + +// Run SWIG on one SWIG input file. +func (b *Builder) swigOne(a *Action, file, objdir string, pcCFLAGS []string, cxx bool, intgosize string) (outGo, outC string, err error) { + p := a.Package + sh := b.Shell(a) + + cgoCPPFLAGS, cgoCFLAGS, cgoCXXFLAGS, _, _, err := b.CFlags(p) + if err != nil { + return "", "", err + } + + var cflags []string + if cxx { + cflags = str.StringList(cgoCPPFLAGS, pcCFLAGS, cgoCXXFLAGS) + } else { + cflags = str.StringList(cgoCPPFLAGS, pcCFLAGS, cgoCFLAGS) + } + + n := 5 // length of ".swig" + if cxx { + n = 8 // length of ".swigcxx" + } + base := file[:len(file)-n] + goFile := base + ".go" + gccBase := base + "_wrap." + gccExt := "c" + if cxx { + gccExt = "cxx" + } + + gccgo := cfg.BuildToolchainName == "gccgo" + + // swig + args := []string{ + "-go", + "-cgo", + "-intgosize", intgosize, + "-module", base, + "-o", objdir + gccBase + gccExt, + "-outdir", objdir, + } + + for _, f := range cflags { + if len(f) > 3 && f[:2] == "-I" { + args = append(args, f) + } + } + + if gccgo { + args = append(args, "-gccgo") + if pkgpath := gccgoPkgpath(p); pkgpath != "" { + args = append(args, "-go-pkgpath", pkgpath) + } + } + if cxx { + args = append(args, "-c++") + } + + out, err := sh.runOut(p.Dir, nil, "swig", args, file) + if err != nil && (bytes.Contains(out, []byte("-intgosize")) || bytes.Contains(out, []byte("-cgo"))) { + return "", "", errors.New("must have SWIG version >= 3.0.6") + } + if err := sh.reportCmd("", "", out, err); err != nil { + return "", "", err + } + + // If the input was x.swig, the output is x.go in the objdir. + // But there might be an x.go in the original dir too, and if it + // uses cgo as well, cgo will be processing both and will + // translate both into x.cgo1.go in the objdir, overwriting one. + // Rename x.go to _x_swig.go to avoid this problem. + // We ignore files in the original dir that begin with underscore + // so _x_swig.go cannot conflict with an original file we were + // going to compile. + goFile = objdir + goFile + newGoFile := objdir + "_" + base + "_swig.go" + if cfg.BuildX || cfg.BuildN { + sh.ShowCmd("", "mv %s %s", goFile, newGoFile) + } + if !cfg.BuildN { + if err := os.Rename(goFile, newGoFile); err != nil { + return "", "", err + } + } + return newGoFile, objdir + gccBase + gccExt, nil +} + +// disableBuildID adjusts a linker command line to avoid creating a +// build ID when creating an object file rather than an executable or +// shared library. Some systems, such as Ubuntu, always add +// --build-id to every link, but we don't want a build ID when we are +// producing an object file. On some of those system a plain -r (not +// -Wl,-r) will turn off --build-id, but clang 3.0 doesn't support a +// plain -r. I don't know how to turn off --build-id when using clang +// other than passing a trailing --build-id=none. So that is what we +// do, but only on systems likely to support it, which is to say, +// systems that normally use gold or the GNU linker. +func (b *Builder) disableBuildID(ldflags []string) []string { + switch cfg.Goos { + case "android", "dragonfly", "linux", "netbsd": + ldflags = append(ldflags, "-Wl,--build-id=none") + } + return ldflags +} + +// mkAbsFiles converts files into a list of absolute files, +// assuming they were originally relative to dir, +// and returns that new list. +func mkAbsFiles(dir string, files []string) []string { + abs := make([]string, len(files)) + for i, f := range files { + if !filepath.IsAbs(f) { + f = filepath.Join(dir, f) + } + abs[i] = f + } + return abs +} + +// passLongArgsInResponseFiles modifies cmd such that, for +// certain programs, long arguments are passed in "response files", a +// file on disk with the arguments, with one arg per line. An actual +// argument starting with '@' means that the rest of the argument is +// a filename of arguments to expand. +// +// See issues 18468 (Windows) and 37768 (Darwin). +func passLongArgsInResponseFiles(cmd *exec.Cmd) (cleanup func()) { + cleanup = func() {} // no cleanup by default + + var argLen int + for _, arg := range cmd.Args { + argLen += len(arg) + } + + // If we're not approaching 32KB of args, just pass args normally. + // (use 30KB instead to be conservative; not sure how accounting is done) + if !useResponseFile(cmd.Path, argLen) { + return + } + + tf, err := os.CreateTemp("", "args") + if err != nil { + log.Fatalf("error writing long arguments to response file: %v", err) + } + cleanup = func() { os.Remove(tf.Name()) } + var buf bytes.Buffer + for _, arg := range cmd.Args[1:] { + fmt.Fprintf(&buf, "%s\n", encodeArg(arg)) + } + if _, err := tf.Write(buf.Bytes()); err != nil { + tf.Close() + cleanup() + log.Fatalf("error writing long arguments to response file: %v", err) + } + if err := tf.Close(); err != nil { + cleanup() + log.Fatalf("error writing long arguments to response file: %v", err) + } + cmd.Args = []string{cmd.Args[0], "@" + tf.Name()} + return cleanup +} + +func useResponseFile(path string, argLen int) bool { + // Unless the program uses objabi.Flagparse, which understands + // response files, don't use response files. + // TODO: Note that other toolchains like CC are missing here for now. + prog := strings.TrimSuffix(filepath.Base(path), ".exe") + switch prog { + case "compile", "link", "cgo", "asm", "cover": + default: + return false + } + + if argLen > sys.ExecArgLengthLimit { + return true + } + + // On the Go build system, use response files about 10% of the + // time, just to exercise this codepath. + isBuilder := os.Getenv("GO_BUILDER_NAME") != "" + if isBuilder && rand.Intn(10) == 0 { + return true + } + + return false +} + +// encodeArg encodes an argument for response file writing. +func encodeArg(arg string) string { + // If there aren't any characters we need to reencode, fastpath out. + if !strings.ContainsAny(arg, "\\\n") { + return arg + } + var b strings.Builder + for _, r := range arg { + switch r { + case '\\': + b.WriteByte('\\') + b.WriteByte('\\') + case '\n': + b.WriteByte('\\') + b.WriteByte('n') + default: + b.WriteRune(r) + } + } + return b.String() +} diff --git a/src/cmd/go/internal/work/exec_test.go b/src/cmd/go/internal/work/exec_test.go new file mode 100644 index 0000000..8bbf25b --- /dev/null +++ b/src/cmd/go/internal/work/exec_test.go @@ -0,0 +1,87 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "bytes" + "cmd/internal/objabi" + "cmd/internal/sys" + "fmt" + "math/rand" + "testing" + "time" + "unicode/utf8" +) + +func TestEncodeArgs(t *testing.T) { + t.Parallel() + tests := []struct { + arg, want string + }{ + {"", ""}, + {"hello", "hello"}, + {"hello\n", "hello\\n"}, + {"hello\\", "hello\\\\"}, + {"hello\nthere", "hello\\nthere"}, + {"\\\n", "\\\\\\n"}, + } + for _, test := range tests { + if got := encodeArg(test.arg); got != test.want { + t.Errorf("encodeArg(%q) = %q, want %q", test.arg, got, test.want) + } + } +} + +func TestEncodeDecode(t *testing.T) { + t.Parallel() + tests := []string{ + "", + "hello", + "hello\\there", + "hello\nthere", + "hello 中国", + "hello \n中\\国", + } + for _, arg := range tests { + if got := objabi.DecodeArg(encodeArg(arg)); got != arg { + t.Errorf("objabi.DecodeArg(encodeArg(%q)) = %q", arg, got) + } + } +} + +func TestEncodeDecodeFuzz(t *testing.T) { + if testing.Short() { + t.Skip("fuzz test is slow") + } + t.Parallel() + + nRunes := sys.ExecArgLengthLimit + 100 + rBuffer := make([]rune, nRunes) + buf := bytes.NewBuffer([]byte(string(rBuffer))) + + seed := time.Now().UnixNano() + t.Logf("rand seed: %v", seed) + rng := rand.New(rand.NewSource(seed)) + + for i := 0; i < 50; i++ { + // Generate a random string of runes. + buf.Reset() + for buf.Len() < sys.ExecArgLengthLimit+1 { + var r rune + for { + r = rune(rng.Intn(utf8.MaxRune + 1)) + if utf8.ValidRune(r) { + break + } + } + fmt.Fprintf(buf, "%c", r) + } + arg := buf.String() + + if got := objabi.DecodeArg(encodeArg(arg)); got != arg { + t.Errorf("[%d] objabi.DecodeArg(encodeArg(%q)) = %q [seed: %v]", i, arg, got, seed) + } + } +} diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go new file mode 100644 index 0000000..e2a5456 --- /dev/null +++ b/src/cmd/go/internal/work/gc.go @@ -0,0 +1,708 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "bufio" + "bytes" + "fmt" + "internal/platform" + "io" + "log" + "os" + "path/filepath" + "runtime" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/gover" + "cmd/go/internal/load" + "cmd/go/internal/str" + "cmd/internal/quoted" + "crypto/sha1" +) + +// Tests can override this by setting $TESTGO_TOOLCHAIN_VERSION. +var ToolchainVersion = runtime.Version() + +// The 'path' used for GOROOT_FINAL when -trimpath is specified +const trimPathGoRootFinal string = "$GOROOT" + +// The Go toolchain. + +type gcToolchain struct{} + +func (gcToolchain) compiler() string { + return base.Tool("compile") +} + +func (gcToolchain) linker() string { + return base.Tool("link") +} + +func pkgPath(a *Action) string { + p := a.Package + ppath := p.ImportPath + if cfg.BuildBuildmode == "plugin" { + ppath = pluginPath(a) + } else if p.Name == "main" && !p.Internal.ForceLibrary { + ppath = "main" + } + return ppath +} + +func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, output []byte, err error) { + p := a.Package + sh := b.Shell(a) + objdir := a.Objdir + if archive != "" { + ofile = archive + } else { + out := "_go_.o" + ofile = objdir + out + } + + pkgpath := pkgPath(a) + defaultGcFlags := []string{"-p", pkgpath} + if p.Module != nil { + v := p.Module.GoVersion + if v == "" { + v = gover.DefaultGoModVersion + } + if allowedVersion(v) { + defaultGcFlags = append(defaultGcFlags, "-lang=go"+gover.Lang(v)) + } + } + if p.Standard { + defaultGcFlags = append(defaultGcFlags, "-std") + } + + // If we're giving the compiler the entire package (no C etc files), tell it that, + // so that it can give good error messages about forward declarations. + // Exceptions: a few standard packages have forward declarations for + // pieces supplied behind-the-scenes by package runtime. + extFiles := len(p.CgoFiles) + len(p.CFiles) + len(p.CXXFiles) + len(p.MFiles) + len(p.FFiles) + len(p.SFiles) + len(p.SysoFiles) + len(p.SwigFiles) + len(p.SwigCXXFiles) + if p.Standard { + switch p.ImportPath { + case "bytes", "internal/poll", "net", "os": + fallthrough + case "runtime/metrics", "runtime/pprof", "runtime/trace": + fallthrough + case "sync", "syscall", "time": + extFiles++ + } + } + if extFiles == 0 { + defaultGcFlags = append(defaultGcFlags, "-complete") + } + if cfg.BuildContext.InstallSuffix != "" { + defaultGcFlags = append(defaultGcFlags, "-installsuffix", cfg.BuildContext.InstallSuffix) + } + if a.buildID != "" { + defaultGcFlags = append(defaultGcFlags, "-buildid", a.buildID) + } + if p.Internal.OmitDebug || cfg.Goos == "plan9" || cfg.Goarch == "wasm" { + defaultGcFlags = append(defaultGcFlags, "-dwarf=false") + } + if strings.HasPrefix(ToolchainVersion, "go1") && !strings.Contains(os.Args[0], "go_bootstrap") { + defaultGcFlags = append(defaultGcFlags, "-goversion", ToolchainVersion) + } + if p.Internal.Cover.Cfg != "" { + defaultGcFlags = append(defaultGcFlags, "-coveragecfg="+p.Internal.Cover.Cfg) + } + if p.Internal.PGOProfile != "" { + defaultGcFlags = append(defaultGcFlags, "-pgoprofile="+p.Internal.PGOProfile) + } + if symabis != "" { + defaultGcFlags = append(defaultGcFlags, "-symabis", symabis) + } + + gcflags := str.StringList(forcedGcflags, p.Internal.Gcflags) + if p.Internal.FuzzInstrument { + gcflags = append(gcflags, fuzzInstrumentFlags()...) + } + // Add -c=N to use concurrent backend compilation, if possible. + if c := gcBackendConcurrency(gcflags); c > 1 { + defaultGcFlags = append(defaultGcFlags, fmt.Sprintf("-c=%d", c)) + } + + args := []any{cfg.BuildToolexec, base.Tool("compile"), "-o", ofile, "-trimpath", a.trimpath(), defaultGcFlags, gcflags} + if p.Internal.LocalPrefix == "" { + args = append(args, "-nolocalimports") + } else { + args = append(args, "-D", p.Internal.LocalPrefix) + } + if importcfg != nil { + if err := sh.writeFile(objdir+"importcfg", importcfg); err != nil { + return "", nil, err + } + args = append(args, "-importcfg", objdir+"importcfg") + } + if embedcfg != nil { + if err := sh.writeFile(objdir+"embedcfg", embedcfg); err != nil { + return "", nil, err + } + args = append(args, "-embedcfg", objdir+"embedcfg") + } + if ofile == archive { + args = append(args, "-pack") + } + if asmhdr { + args = append(args, "-asmhdr", objdir+"go_asm.h") + } + + for _, f := range gofiles { + f := mkAbs(p.Dir, f) + + // Handle overlays. Convert path names using OverlayPath + // so these paths can be handed directly to tools. + // Deleted files won't show up in when scanning directories earlier, + // so OverlayPath will never return "" (meaning a deleted file) here. + // TODO(#39958): Handle cases where the package directory + // doesn't exist on disk (this can happen when all the package's + // files are in an overlay): the code expects the package directory + // to exist and runs some tools in that directory. + // TODO(#39958): Process the overlays when the + // gofiles, cgofiles, cfiles, sfiles, and cxxfiles variables are + // created in (*Builder).build. Doing that requires rewriting the + // code that uses those values to expect absolute paths. + f, _ = fsys.OverlayPath(f) + + args = append(args, f) + } + + output, err = sh.runOut(base.Cwd(), nil, args...) + return ofile, output, err +} + +// gcBackendConcurrency returns the backend compiler concurrency level for a package compilation. +func gcBackendConcurrency(gcflags []string) int { + // First, check whether we can use -c at all for this compilation. + canDashC := concurrentGCBackendCompilationEnabledByDefault + + switch e := os.Getenv("GO19CONCURRENTCOMPILATION"); e { + case "0": + canDashC = false + case "1": + canDashC = true + case "": + // Not set. Use default. + default: + log.Fatalf("GO19CONCURRENTCOMPILATION must be 0, 1, or unset, got %q", e) + } + + // TODO: Test and delete these conditions. + if cfg.ExperimentErr != nil || cfg.Experiment.FieldTrack || cfg.Experiment.PreemptibleLoops { + canDashC = false + } + + if !canDashC { + return 1 + } + + // Decide how many concurrent backend compilations to allow. + // + // If we allow too many, in theory we might end up with p concurrent processes, + // each with c concurrent backend compiles, all fighting over the same resources. + // However, in practice, that seems not to happen too much. + // Most build graphs are surprisingly serial, so p==1 for much of the build. + // Furthermore, concurrent backend compilation is only enabled for a part + // of the overall compiler execution, so c==1 for much of the build. + // So don't worry too much about that interaction for now. + // + // However, in practice, setting c above 4 tends not to help very much. + // See the analysis in CL 41192. + // + // TODO(josharian): attempt to detect whether this particular compilation + // is likely to be a bottleneck, e.g. when: + // - it has no successor packages to compile (usually package main) + // - all paths through the build graph pass through it + // - critical path scheduling says it is high priority + // and in such a case, set c to runtime.GOMAXPROCS(0). + // By default this is the same as runtime.NumCPU. + // We do this now when p==1. + // To limit parallelism, set GOMAXPROCS below numCPU; this may be useful + // on a low-memory builder, or if a deterministic build order is required. + c := runtime.GOMAXPROCS(0) + if cfg.BuildP == 1 { + // No process parallelism, do not cap compiler parallelism. + return c + } + // Some process parallelism. Set c to min(4, maxprocs). + if c > 4 { + c = 4 + } + return c +} + +// trimpath returns the -trimpath argument to use +// when compiling the action. +func (a *Action) trimpath() string { + // Keep in sync with Builder.ccompile + // The trimmed paths are a little different, but we need to trim in the + // same situations. + + // Strip the object directory entirely. + objdir := a.Objdir + if len(objdir) > 1 && objdir[len(objdir)-1] == filepath.Separator { + objdir = objdir[:len(objdir)-1] + } + rewrite := "" + + rewriteDir := a.Package.Dir + if cfg.BuildTrimpath { + importPath := a.Package.Internal.OrigImportPath + if m := a.Package.Module; m != nil && m.Version != "" { + rewriteDir = m.Path + "@" + m.Version + strings.TrimPrefix(importPath, m.Path) + } else { + rewriteDir = importPath + } + rewrite += a.Package.Dir + "=>" + rewriteDir + ";" + } + + // Add rewrites for overlays. The 'from' and 'to' paths in overlays don't need to have + // same basename, so go from the overlay contents file path (passed to the compiler) + // to the path the disk path would be rewritten to. + + cgoFiles := make(map[string]bool) + for _, f := range a.Package.CgoFiles { + cgoFiles[f] = true + } + + // TODO(matloob): Higher up in the stack, when the logic for deciding when to make copies + // of c/c++/m/f/hfiles is consolidated, use the same logic that Build uses to determine + // whether to create the copies in objdir to decide whether to rewrite objdir to the + // package directory here. + var overlayNonGoRewrites string // rewrites for non-go files + hasCgoOverlay := false + if fsys.OverlayFile != "" { + for _, filename := range a.Package.AllFiles() { + path := filename + if !filepath.IsAbs(path) { + path = filepath.Join(a.Package.Dir, path) + } + base := filepath.Base(path) + isGo := strings.HasSuffix(filename, ".go") || strings.HasSuffix(filename, ".s") + isCgo := cgoFiles[filename] || !isGo + overlayPath, isOverlay := fsys.OverlayPath(path) + if isCgo && isOverlay { + hasCgoOverlay = true + } + if !isCgo && isOverlay { + rewrite += overlayPath + "=>" + filepath.Join(rewriteDir, base) + ";" + } else if isCgo { + // Generate rewrites for non-Go files copied to files in objdir. + if filepath.Dir(path) == a.Package.Dir { + // This is a file copied to objdir. + overlayNonGoRewrites += filepath.Join(objdir, base) + "=>" + filepath.Join(rewriteDir, base) + ";" + } + } else { + // Non-overlay Go files are covered by the a.Package.Dir rewrite rule above. + } + } + } + if hasCgoOverlay { + rewrite += overlayNonGoRewrites + } + rewrite += objdir + "=>" + + return rewrite +} + +func asmArgs(a *Action, p *load.Package) []any { + // Add -I pkg/GOOS_GOARCH so #include "textflag.h" works in .s files. + inc := filepath.Join(cfg.GOROOT, "pkg", "include") + pkgpath := pkgPath(a) + args := []any{cfg.BuildToolexec, base.Tool("asm"), "-p", pkgpath, "-trimpath", a.trimpath(), "-I", a.Objdir, "-I", inc, "-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch, forcedAsmflags, p.Internal.Asmflags} + if p.ImportPath == "runtime" && cfg.Goarch == "386" { + for _, arg := range forcedAsmflags { + if arg == "-dynlink" { + args = append(args, "-D=GOBUILDMODE_shared=1") + } + } + } + + if cfg.Goarch == "386" { + // Define GO386_value from cfg.GO386. + args = append(args, "-D", "GO386_"+cfg.GO386) + } + + if cfg.Goarch == "amd64" { + // Define GOAMD64_value from cfg.GOAMD64. + args = append(args, "-D", "GOAMD64_"+cfg.GOAMD64) + } + + if cfg.Goarch == "mips" || cfg.Goarch == "mipsle" { + // Define GOMIPS_value from cfg.GOMIPS. + args = append(args, "-D", "GOMIPS_"+cfg.GOMIPS) + } + + if cfg.Goarch == "mips64" || cfg.Goarch == "mips64le" { + // Define GOMIPS64_value from cfg.GOMIPS64. + args = append(args, "-D", "GOMIPS64_"+cfg.GOMIPS64) + } + + if cfg.Goarch == "ppc64" || cfg.Goarch == "ppc64le" { + // Define GOPPC64_power8..N from cfg.PPC64. + // We treat each powerpc version as a superset of functionality. + switch cfg.GOPPC64 { + case "power10": + args = append(args, "-D", "GOPPC64_power10") + fallthrough + case "power9": + args = append(args, "-D", "GOPPC64_power9") + fallthrough + default: // This should always be power8. + args = append(args, "-D", "GOPPC64_power8") + } + } + + if cfg.Goarch == "arm" { + // Define GOARM_value from cfg.GOARM. + switch cfg.GOARM { + case "7": + args = append(args, "-D", "GOARM_7") + fallthrough + case "6": + args = append(args, "-D", "GOARM_6") + fallthrough + default: + args = append(args, "-D", "GOARM_5") + } + } + + return args +} + +func (gcToolchain) asm(b *Builder, a *Action, sfiles []string) ([]string, error) { + p := a.Package + args := asmArgs(a, p) + + var ofiles []string + for _, sfile := range sfiles { + overlayPath, _ := fsys.OverlayPath(mkAbs(p.Dir, sfile)) + ofile := a.Objdir + sfile[:len(sfile)-len(".s")] + ".o" + ofiles = append(ofiles, ofile) + args1 := append(args, "-o", ofile, overlayPath) + if err := b.Shell(a).run(p.Dir, p.ImportPath, nil, args1...); err != nil { + return nil, err + } + } + return ofiles, nil +} + +func (gcToolchain) symabis(b *Builder, a *Action, sfiles []string) (string, error) { + sh := b.Shell(a) + + mkSymabis := func(p *load.Package, sfiles []string, path string) error { + args := asmArgs(a, p) + args = append(args, "-gensymabis", "-o", path) + for _, sfile := range sfiles { + if p.ImportPath == "runtime/cgo" && strings.HasPrefix(sfile, "gcc_") { + continue + } + op, _ := fsys.OverlayPath(mkAbs(p.Dir, sfile)) + args = append(args, op) + } + + // Supply an empty go_asm.h as if the compiler had been run. + // -gensymabis parsing is lax enough that we don't need the + // actual definitions that would appear in go_asm.h. + if err := sh.writeFile(a.Objdir+"go_asm.h", nil); err != nil { + return err + } + + return sh.run(p.Dir, p.ImportPath, nil, args...) + } + + var symabis string // Only set if we actually create the file + p := a.Package + if len(sfiles) != 0 { + symabis = a.Objdir + "symabis" + if err := mkSymabis(p, sfiles, symabis); err != nil { + return "", err + } + } + + return symabis, nil +} + +// toolVerify checks that the command line args writes the same output file +// if run using newTool instead. +// Unused now but kept around for future use. +func toolVerify(a *Action, b *Builder, p *load.Package, newTool string, ofile string, args []any) error { + newArgs := make([]any, len(args)) + copy(newArgs, args) + newArgs[1] = base.Tool(newTool) + newArgs[3] = ofile + ".new" // x.6 becomes x.6.new + if err := b.Shell(a).run(p.Dir, p.ImportPath, nil, newArgs...); err != nil { + return err + } + data1, err := os.ReadFile(ofile) + if err != nil { + return err + } + data2, err := os.ReadFile(ofile + ".new") + if err != nil { + return err + } + if !bytes.Equal(data1, data2) { + return fmt.Errorf("%s and %s produced different output files:\n%s\n%s", filepath.Base(args[1].(string)), newTool, strings.Join(str.StringList(args...), " "), strings.Join(str.StringList(newArgs...), " ")) + } + os.Remove(ofile + ".new") + return nil +} + +func (gcToolchain) pack(b *Builder, a *Action, afile string, ofiles []string) error { + var absOfiles []string + for _, f := range ofiles { + absOfiles = append(absOfiles, mkAbs(a.Objdir, f)) + } + absAfile := mkAbs(a.Objdir, afile) + + // The archive file should have been created by the compiler. + // Since it used to not work that way, verify. + if !cfg.BuildN { + if _, err := os.Stat(absAfile); err != nil { + base.Fatalf("os.Stat of archive file failed: %v", err) + } + } + + p := a.Package + sh := b.Shell(a) + if cfg.BuildN || cfg.BuildX { + cmdline := str.StringList(base.Tool("pack"), "r", absAfile, absOfiles) + sh.ShowCmd(p.Dir, "%s # internal", joinUnambiguously(cmdline)) + } + if cfg.BuildN { + return nil + } + if err := packInternal(absAfile, absOfiles); err != nil { + return sh.reportCmd("", "", nil, err) + } + return nil +} + +func packInternal(afile string, ofiles []string) error { + dst, err := os.OpenFile(afile, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + return err + } + defer dst.Close() // only for error returns or panics + w := bufio.NewWriter(dst) + + for _, ofile := range ofiles { + src, err := os.Open(ofile) + if err != nil { + return err + } + fi, err := src.Stat() + if err != nil { + src.Close() + return err + } + // Note: Not using %-16.16s format because we care + // about bytes, not runes. + name := fi.Name() + if len(name) > 16 { + name = name[:16] + } else { + name += strings.Repeat(" ", 16-len(name)) + } + size := fi.Size() + fmt.Fprintf(w, "%s%-12d%-6d%-6d%-8o%-10d`\n", + name, 0, 0, 0, 0644, size) + n, err := io.Copy(w, src) + src.Close() + if err == nil && n < size { + err = io.ErrUnexpectedEOF + } else if err == nil && n > size { + err = fmt.Errorf("file larger than size reported by stat") + } + if err != nil { + return fmt.Errorf("copying %s to %s: %v", ofile, afile, err) + } + if size&1 != 0 { + w.WriteByte(0) + } + } + + if err := w.Flush(); err != nil { + return err + } + return dst.Close() +} + +// setextld sets the appropriate linker flags for the specified compiler. +func setextld(ldflags []string, compiler []string) ([]string, error) { + for _, f := range ldflags { + if f == "-extld" || strings.HasPrefix(f, "-extld=") { + // don't override -extld if supplied + return ldflags, nil + } + } + joined, err := quoted.Join(compiler) + if err != nil { + return nil, err + } + return append(ldflags, "-extld="+joined), nil +} + +// pluginPath computes the package path for a plugin main package. +// +// This is typically the import path of the main package p, unless the +// plugin is being built directly from source files. In that case we +// combine the package build ID with the contents of the main package +// source files. This allows us to identify two different plugins +// built from two source files with the same name. +func pluginPath(a *Action) string { + p := a.Package + if p.ImportPath != "command-line-arguments" { + return p.ImportPath + } + h := sha1.New() + buildID := a.buildID + if a.Mode == "link" { + // For linking, use the main package's build ID instead of + // the binary's build ID, so it is the same hash used in + // compiling and linking. + // When compiling, we use actionID/actionID (instead of + // actionID/contentID) as a temporary build ID to compute + // the hash. Do the same here. (See buildid.go:useCache) + // The build ID matters because it affects the overall hash + // in the plugin's pseudo-import path returned below. + // We need to use the same import path when compiling and linking. + id := strings.Split(buildID, buildIDSeparator) + buildID = id[1] + buildIDSeparator + id[1] + } + fmt.Fprintf(h, "build ID: %s\n", buildID) + for _, file := range str.StringList(p.GoFiles, p.CgoFiles, p.SFiles) { + data, err := os.ReadFile(filepath.Join(p.Dir, file)) + if err != nil { + base.Fatalf("go: %s", err) + } + h.Write(data) + } + return fmt.Sprintf("plugin/unnamed-%x", h.Sum(nil)) +} + +func (gcToolchain) ld(b *Builder, root *Action, targetPath, importcfg, mainpkg string) error { + cxx := len(root.Package.CXXFiles) > 0 || len(root.Package.SwigCXXFiles) > 0 + for _, a := range root.Deps { + if a.Package != nil && (len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0) { + cxx = true + } + } + var ldflags []string + if cfg.BuildContext.InstallSuffix != "" { + ldflags = append(ldflags, "-installsuffix", cfg.BuildContext.InstallSuffix) + } + if root.Package.Internal.OmitDebug { + ldflags = append(ldflags, "-s", "-w") + } + if cfg.BuildBuildmode == "plugin" { + ldflags = append(ldflags, "-pluginpath", pluginPath(root)) + } + + // Store BuildID inside toolchain binaries as a unique identifier of the + // tool being run, for use by content-based staleness determination. + if root.Package.Goroot && strings.HasPrefix(root.Package.ImportPath, "cmd/") { + // External linking will include our build id in the external + // linker's build id, which will cause our build id to not + // match the next time the tool is built. + // Rely on the external build id instead. + if !platform.MustLinkExternal(cfg.Goos, cfg.Goarch, false) { + ldflags = append(ldflags, "-X=cmd/internal/objabi.buildID="+root.buildID) + } + } + + // Store default GODEBUG in binaries. + if root.Package.DefaultGODEBUG != "" { + ldflags = append(ldflags, "-X=runtime.godebugDefault="+root.Package.DefaultGODEBUG) + } + + // If the user has not specified the -extld option, then specify the + // appropriate linker. In case of C++ code, use the compiler named + // by the CXX environment variable or defaultCXX if CXX is not set. + // Else, use the CC environment variable and defaultCC as fallback. + var compiler []string + if cxx { + compiler = envList("CXX", cfg.DefaultCXX(cfg.Goos, cfg.Goarch)) + } else { + compiler = envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch)) + } + ldflags = append(ldflags, "-buildmode="+ldBuildmode) + if root.buildID != "" { + ldflags = append(ldflags, "-buildid="+root.buildID) + } + ldflags = append(ldflags, forcedLdflags...) + ldflags = append(ldflags, root.Package.Internal.Ldflags...) + ldflags, err := setextld(ldflags, compiler) + if err != nil { + return err + } + + // On OS X when using external linking to build a shared library, + // the argument passed here to -o ends up recorded in the final + // shared library in the LC_ID_DYLIB load command. + // To avoid putting the temporary output directory name there + // (and making the resulting shared library useless), + // run the link in the output directory so that -o can name + // just the final path element. + // On Windows, DLL file name is recorded in PE file + // export section, so do like on OS X. + // On Linux, for a shared object, at least with the Gold linker, + // the output file path is recorded in the .gnu.version_d section. + dir := "." + if cfg.BuildBuildmode == "c-shared" || cfg.BuildBuildmode == "plugin" { + dir, targetPath = filepath.Split(targetPath) + } + + env := []string{} + if cfg.BuildTrimpath { + env = append(env, "GOROOT_FINAL="+trimPathGoRootFinal) + } + return b.Shell(root).run(dir, root.Package.ImportPath, env, cfg.BuildToolexec, base.Tool("link"), "-o", targetPath, "-importcfg", importcfg, ldflags, mainpkg) +} + +func (gcToolchain) ldShared(b *Builder, root *Action, toplevelactions []*Action, targetPath, importcfg string, allactions []*Action) error { + ldflags := []string{"-installsuffix", cfg.BuildContext.InstallSuffix} + ldflags = append(ldflags, "-buildmode=shared") + ldflags = append(ldflags, forcedLdflags...) + ldflags = append(ldflags, root.Package.Internal.Ldflags...) + cxx := false + for _, a := range allactions { + if a.Package != nil && (len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0) { + cxx = true + } + } + // If the user has not specified the -extld option, then specify the + // appropriate linker. In case of C++ code, use the compiler named + // by the CXX environment variable or defaultCXX if CXX is not set. + // Else, use the CC environment variable and defaultCC as fallback. + var compiler []string + if cxx { + compiler = envList("CXX", cfg.DefaultCXX(cfg.Goos, cfg.Goarch)) + } else { + compiler = envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch)) + } + ldflags, err := setextld(ldflags, compiler) + if err != nil { + return err + } + for _, d := range toplevelactions { + if !strings.HasSuffix(d.Target, ".a") { // omit unsafe etc and actions for other shared libraries + continue + } + ldflags = append(ldflags, d.Package.ImportPath+"="+d.Target) + } + return b.Shell(root).run(".", targetPath, nil, cfg.BuildToolexec, base.Tool("link"), "-o", targetPath, "-importcfg", importcfg, ldflags) +} + +func (gcToolchain) cc(b *Builder, a *Action, ofile, cfile string) error { + return fmt.Errorf("%s: C source files not supported without cgo", mkAbs(a.Package.Dir, cfile)) +} diff --git a/src/cmd/go/internal/work/gccgo.go b/src/cmd/go/internal/work/gccgo.go new file mode 100644 index 0000000..2dce9f1 --- /dev/null +++ b/src/cmd/go/internal/work/gccgo.go @@ -0,0 +1,672 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/load" + "cmd/go/internal/str" + "cmd/internal/pkgpath" +) + +// The Gccgo toolchain. + +type gccgoToolchain struct{} + +var GccgoName, GccgoBin string +var gccgoErr error + +func init() { + GccgoName = cfg.Getenv("GCCGO") + if GccgoName == "" { + GccgoName = "gccgo" + } + GccgoBin, gccgoErr = cfg.LookPath(GccgoName) +} + +func (gccgoToolchain) compiler() string { + checkGccgoBin() + return GccgoBin +} + +func (gccgoToolchain) linker() string { + checkGccgoBin() + return GccgoBin +} + +func (gccgoToolchain) ar() []string { + return envList("AR", "ar") +} + +func checkGccgoBin() { + if gccgoErr == nil { + return + } + fmt.Fprintf(os.Stderr, "cmd/go: gccgo: %s\n", gccgoErr) + base.SetExitStatus(2) + base.Exit() +} + +func (tools gccgoToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, output []byte, err error) { + p := a.Package + sh := b.Shell(a) + objdir := a.Objdir + out := "_go_.o" + ofile = objdir + out + gcargs := []string{"-g"} + gcargs = append(gcargs, b.gccArchArgs()...) + gcargs = append(gcargs, "-fdebug-prefix-map="+b.WorkDir+"=/tmp/go-build") + gcargs = append(gcargs, "-gno-record-gcc-switches") + if pkgpath := gccgoPkgpath(p); pkgpath != "" { + gcargs = append(gcargs, "-fgo-pkgpath="+pkgpath) + } + if p.Internal.LocalPrefix != "" { + gcargs = append(gcargs, "-fgo-relative-import-path="+p.Internal.LocalPrefix) + } + + args := str.StringList(tools.compiler(), "-c", gcargs, "-o", ofile, forcedGccgoflags) + if importcfg != nil { + if b.gccSupportsFlag(args[:1], "-fgo-importcfg=/dev/null") { + if err := sh.writeFile(objdir+"importcfg", importcfg); err != nil { + return "", nil, err + } + args = append(args, "-fgo-importcfg="+objdir+"importcfg") + } else { + root := objdir + "_importcfgroot_" + if err := buildImportcfgSymlinks(sh, root, importcfg); err != nil { + return "", nil, err + } + args = append(args, "-I", root) + } + } + if embedcfg != nil && b.gccSupportsFlag(args[:1], "-fgo-embedcfg=/dev/null") { + if err := sh.writeFile(objdir+"embedcfg", embedcfg); err != nil { + return "", nil, err + } + args = append(args, "-fgo-embedcfg="+objdir+"embedcfg") + } + + if b.gccSupportsFlag(args[:1], "-ffile-prefix-map=a=b") { + if cfg.BuildTrimpath { + args = append(args, "-ffile-prefix-map="+base.Cwd()+"=.") + args = append(args, "-ffile-prefix-map="+b.WorkDir+"=/tmp/go-build") + } + if fsys.OverlayFile != "" { + for _, name := range gofiles { + absPath := mkAbs(p.Dir, name) + overlayPath, ok := fsys.OverlayPath(absPath) + if !ok { + continue + } + toPath := absPath + // gccgo only applies the last matching rule, so also handle the case where + // BuildTrimpath is true and the path is relative to base.Cwd(). + if cfg.BuildTrimpath && str.HasFilePathPrefix(toPath, base.Cwd()) { + toPath = "." + toPath[len(base.Cwd()):] + } + args = append(args, "-ffile-prefix-map="+overlayPath+"="+toPath) + } + } + } + + args = append(args, a.Package.Internal.Gccgoflags...) + for _, f := range gofiles { + f := mkAbs(p.Dir, f) + // Overlay files if necessary. + // See comment on gctoolchain.gc about overlay TODOs + f, _ = fsys.OverlayPath(f) + args = append(args, f) + } + + output, err = sh.runOut(p.Dir, nil, args) + return ofile, output, err +} + +// buildImportcfgSymlinks builds in root a tree of symlinks +// implementing the directives from importcfg. +// This serves as a temporary transition mechanism until +// we can depend on gccgo reading an importcfg directly. +// (The Go 1.9 and later gc compilers already do.) +func buildImportcfgSymlinks(sh *Shell, root string, importcfg []byte) error { + for lineNum, line := range strings.Split(string(importcfg), "\n") { + lineNum++ // 1-based + line = strings.TrimSpace(line) + if line == "" { + continue + } + if line == "" || strings.HasPrefix(line, "#") { + continue + } + var verb, args string + if i := strings.Index(line, " "); i < 0 { + verb = line + } else { + verb, args = line[:i], strings.TrimSpace(line[i+1:]) + } + before, after, _ := strings.Cut(args, "=") + switch verb { + default: + base.Fatalf("importcfg:%d: unknown directive %q", lineNum, verb) + case "packagefile": + if before == "" || after == "" { + return fmt.Errorf(`importcfg:%d: invalid packagefile: syntax is "packagefile path=filename": %s`, lineNum, line) + } + archive := gccgoArchive(root, before) + if err := sh.Mkdir(filepath.Dir(archive)); err != nil { + return err + } + if err := sh.Symlink(after, archive); err != nil { + return err + } + case "importmap": + if before == "" || after == "" { + return fmt.Errorf(`importcfg:%d: invalid importmap: syntax is "importmap old=new": %s`, lineNum, line) + } + beforeA := gccgoArchive(root, before) + afterA := gccgoArchive(root, after) + if err := sh.Mkdir(filepath.Dir(beforeA)); err != nil { + return err + } + if err := sh.Mkdir(filepath.Dir(afterA)); err != nil { + return err + } + if err := sh.Symlink(afterA, beforeA); err != nil { + return err + } + case "packageshlib": + return fmt.Errorf("gccgo -importcfg does not support shared libraries") + } + } + return nil +} + +func (tools gccgoToolchain) asm(b *Builder, a *Action, sfiles []string) ([]string, error) { + p := a.Package + var ofiles []string + for _, sfile := range sfiles { + base := filepath.Base(sfile) + ofile := a.Objdir + base[:len(base)-len(".s")] + ".o" + ofiles = append(ofiles, ofile) + sfile, _ = fsys.OverlayPath(mkAbs(p.Dir, sfile)) + defs := []string{"-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch} + if pkgpath := tools.gccgoCleanPkgpath(b, p); pkgpath != "" { + defs = append(defs, `-D`, `GOPKGPATH=`+pkgpath) + } + defs = tools.maybePIC(defs) + defs = append(defs, b.gccArchArgs()...) + err := b.Shell(a).run(p.Dir, p.ImportPath, nil, tools.compiler(), "-xassembler-with-cpp", "-I", a.Objdir, "-c", "-o", ofile, defs, sfile) + if err != nil { + return nil, err + } + } + return ofiles, nil +} + +func (gccgoToolchain) symabis(b *Builder, a *Action, sfiles []string) (string, error) { + return "", nil +} + +func gccgoArchive(basedir, imp string) string { + end := filepath.FromSlash(imp + ".a") + afile := filepath.Join(basedir, end) + // add "lib" to the final element + return filepath.Join(filepath.Dir(afile), "lib"+filepath.Base(afile)) +} + +func (tools gccgoToolchain) pack(b *Builder, a *Action, afile string, ofiles []string) error { + p := a.Package + sh := b.Shell(a) + objdir := a.Objdir + var absOfiles []string + for _, f := range ofiles { + absOfiles = append(absOfiles, mkAbs(objdir, f)) + } + var arArgs []string + if cfg.Goos == "aix" && cfg.Goarch == "ppc64" { + // AIX puts both 32-bit and 64-bit objects in the same archive. + // Tell the AIX "ar" command to only care about 64-bit objects. + arArgs = []string{"-X64"} + } + absAfile := mkAbs(objdir, afile) + // Try with D modifier first, then without if that fails. + output, err := sh.runOut(p.Dir, nil, tools.ar(), arArgs, "rcD", absAfile, absOfiles) + if err != nil { + return sh.run(p.Dir, p.ImportPath, nil, tools.ar(), arArgs, "rc", absAfile, absOfiles) + } + + // Show the output if there is any even without errors. + return sh.reportCmd("", "", output, nil) +} + +func (tools gccgoToolchain) link(b *Builder, root *Action, out, importcfg string, allactions []*Action, buildmode, desc string) error { + sh := b.Shell(root) + + // gccgo needs explicit linking with all package dependencies, + // and all LDFLAGS from cgo dependencies. + afiles := []string{} + shlibs := []string{} + ldflags := b.gccArchArgs() + cgoldflags := []string{} + usesCgo := false + cxx := false + objc := false + fortran := false + if root.Package != nil { + cxx = len(root.Package.CXXFiles) > 0 || len(root.Package.SwigCXXFiles) > 0 + objc = len(root.Package.MFiles) > 0 + fortran = len(root.Package.FFiles) > 0 + } + + readCgoFlags := func(flagsFile string) error { + flags, err := os.ReadFile(flagsFile) + if err != nil { + return err + } + const ldflagsPrefix = "_CGO_LDFLAGS=" + for _, line := range strings.Split(string(flags), "\n") { + if strings.HasPrefix(line, ldflagsPrefix) { + flag := line[len(ldflagsPrefix):] + // Every _cgo_flags file has -g and -O2 in _CGO_LDFLAGS + // but they don't mean anything to the linker so filter + // them out. + if flag != "-g" && !strings.HasPrefix(flag, "-O") { + cgoldflags = append(cgoldflags, flag) + } + } + } + return nil + } + + var arArgs []string + if cfg.Goos == "aix" && cfg.Goarch == "ppc64" { + // AIX puts both 32-bit and 64-bit objects in the same archive. + // Tell the AIX "ar" command to only care about 64-bit objects. + arArgs = []string{"-X64"} + } + + newID := 0 + readAndRemoveCgoFlags := func(archive string) (string, error) { + newID++ + newArchive := root.Objdir + fmt.Sprintf("_pkg%d_.a", newID) + if err := sh.CopyFile(newArchive, archive, 0666, false); err != nil { + return "", err + } + if cfg.BuildN || cfg.BuildX { + sh.ShowCmd("", "ar d %s _cgo_flags", newArchive) + if cfg.BuildN { + // TODO(rsc): We could do better about showing the right _cgo_flags even in -n mode. + // Either the archive is already built and we can read them out, + // or we're printing commands to build the archive and can + // forward the _cgo_flags directly to this step. + return "", nil + } + } + err := sh.run(root.Objdir, desc, nil, tools.ar(), arArgs, "x", newArchive, "_cgo_flags") + if err != nil { + return "", err + } + err = sh.run(".", desc, nil, tools.ar(), arArgs, "d", newArchive, "_cgo_flags") + if err != nil { + return "", err + } + err = readCgoFlags(filepath.Join(root.Objdir, "_cgo_flags")) + if err != nil { + return "", err + } + return newArchive, nil + } + + // If using -linkshared, find the shared library deps. + haveShlib := make(map[string]bool) + targetBase := filepath.Base(root.Target) + if cfg.BuildLinkshared { + for _, a := range root.Deps { + p := a.Package + if p == nil || p.Shlib == "" { + continue + } + + // The .a we are linking into this .so + // will have its Shlib set to this .so. + // Don't start thinking we want to link + // this .so into itself. + base := filepath.Base(p.Shlib) + if base != targetBase { + haveShlib[base] = true + } + } + } + + // Arrange the deps into afiles and shlibs. + addedShlib := make(map[string]bool) + for _, a := range root.Deps { + p := a.Package + if p != nil && p.Shlib != "" && haveShlib[filepath.Base(p.Shlib)] { + // This is a package linked into a shared + // library that we will put into shlibs. + continue + } + + if haveShlib[filepath.Base(a.Target)] { + // This is a shared library we want to link against. + if !addedShlib[a.Target] { + shlibs = append(shlibs, a.Target) + addedShlib[a.Target] = true + } + continue + } + + if p != nil { + target := a.built + if p.UsesCgo() || p.UsesSwig() { + var err error + target, err = readAndRemoveCgoFlags(target) + if err != nil { + continue + } + } + + afiles = append(afiles, target) + } + } + + for _, a := range allactions { + if a.Package == nil { + continue + } + if len(a.Package.CgoFiles) > 0 { + usesCgo = true + } + if a.Package.UsesSwig() { + usesCgo = true + } + if len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0 { + cxx = true + } + if len(a.Package.MFiles) > 0 { + objc = true + } + if len(a.Package.FFiles) > 0 { + fortran = true + } + } + + wholeArchive := []string{"-Wl,--whole-archive"} + noWholeArchive := []string{"-Wl,--no-whole-archive"} + if cfg.Goos == "aix" { + wholeArchive = nil + noWholeArchive = nil + } + ldflags = append(ldflags, wholeArchive...) + ldflags = append(ldflags, afiles...) + ldflags = append(ldflags, noWholeArchive...) + + ldflags = append(ldflags, cgoldflags...) + ldflags = append(ldflags, envList("CGO_LDFLAGS", "")...) + if cfg.Goos != "aix" { + ldflags = str.StringList("-Wl,-(", ldflags, "-Wl,-)") + } + + if root.buildID != "" { + // On systems that normally use gold or the GNU linker, + // use the --build-id option to write a GNU build ID note. + switch cfg.Goos { + case "android", "dragonfly", "linux", "netbsd": + ldflags = append(ldflags, fmt.Sprintf("-Wl,--build-id=0x%x", root.buildID)) + } + } + + var rLibPath string + if cfg.Goos == "aix" { + rLibPath = "-Wl,-blibpath=" + } else { + rLibPath = "-Wl,-rpath=" + } + for _, shlib := range shlibs { + ldflags = append( + ldflags, + "-L"+filepath.Dir(shlib), + rLibPath+filepath.Dir(shlib), + "-l"+strings.TrimSuffix( + strings.TrimPrefix(filepath.Base(shlib), "lib"), + ".so")) + } + + var realOut string + goLibBegin := str.StringList(wholeArchive, "-lgolibbegin", noWholeArchive) + switch buildmode { + case "exe": + if usesCgo && cfg.Goos == "linux" { + ldflags = append(ldflags, "-Wl,-E") + } + + case "c-archive": + // Link the Go files into a single .o, and also link + // in -lgolibbegin. + // + // We need to use --whole-archive with -lgolibbegin + // because it doesn't define any symbols that will + // cause the contents to be pulled in; it's just + // initialization code. + // + // The user remains responsible for linking against + // -lgo -lpthread -lm in the final link. We can't use + // -r to pick them up because we can't combine + // split-stack and non-split-stack code in a single -r + // link, and libgo picks up non-split-stack code from + // libffi. + ldflags = append(ldflags, "-Wl,-r", "-nostdlib") + ldflags = append(ldflags, goLibBegin...) + + if nopie := b.gccNoPie([]string{tools.linker()}); nopie != "" { + ldflags = append(ldflags, nopie) + } + + // We are creating an object file, so we don't want a build ID. + if root.buildID == "" { + ldflags = b.disableBuildID(ldflags) + } + + realOut = out + out = out + ".o" + + case "c-shared": + ldflags = append(ldflags, "-shared", "-nostdlib") + ldflags = append(ldflags, goLibBegin...) + ldflags = append(ldflags, "-lgo", "-lgcc_s", "-lgcc", "-lc", "-lgcc") + + case "shared": + if cfg.Goos != "aix" { + ldflags = append(ldflags, "-zdefs") + } + ldflags = append(ldflags, "-shared", "-nostdlib", "-lgo", "-lgcc_s", "-lgcc", "-lc") + + default: + base.Fatalf("-buildmode=%s not supported for gccgo", buildmode) + } + + switch buildmode { + case "exe", "c-shared": + if cxx { + ldflags = append(ldflags, "-lstdc++") + } + if objc { + ldflags = append(ldflags, "-lobjc") + } + if fortran { + fc := cfg.Getenv("FC") + if fc == "" { + fc = "gfortran" + } + // support gfortran out of the box and let others pass the correct link options + // via CGO_LDFLAGS + if strings.Contains(fc, "gfortran") { + ldflags = append(ldflags, "-lgfortran") + } + } + } + + if err := sh.run(".", desc, nil, tools.linker(), "-o", out, ldflags, forcedGccgoflags, root.Package.Internal.Gccgoflags); err != nil { + return err + } + + switch buildmode { + case "c-archive": + if err := sh.run(".", desc, nil, tools.ar(), arArgs, "rc", realOut, out); err != nil { + return err + } + } + return nil +} + +func (tools gccgoToolchain) ld(b *Builder, root *Action, targetPath, importcfg, mainpkg string) error { + return tools.link(b, root, targetPath, importcfg, root.Deps, ldBuildmode, root.Package.ImportPath) +} + +func (tools gccgoToolchain) ldShared(b *Builder, root *Action, toplevelactions []*Action, targetPath, importcfg string, allactions []*Action) error { + return tools.link(b, root, targetPath, importcfg, allactions, "shared", targetPath) +} + +func (tools gccgoToolchain) cc(b *Builder, a *Action, ofile, cfile string) error { + p := a.Package + inc := filepath.Join(cfg.GOROOT, "pkg", "include") + cfile = mkAbs(p.Dir, cfile) + defs := []string{"-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch} + defs = append(defs, b.gccArchArgs()...) + if pkgpath := tools.gccgoCleanPkgpath(b, p); pkgpath != "" { + defs = append(defs, `-D`, `GOPKGPATH="`+pkgpath+`"`) + } + compiler := envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch)) + if b.gccSupportsFlag(compiler, "-fsplit-stack") { + defs = append(defs, "-fsplit-stack") + } + defs = tools.maybePIC(defs) + if b.gccSupportsFlag(compiler, "-ffile-prefix-map=a=b") { + defs = append(defs, "-ffile-prefix-map="+base.Cwd()+"=.") + defs = append(defs, "-ffile-prefix-map="+b.WorkDir+"=/tmp/go-build") + } else if b.gccSupportsFlag(compiler, "-fdebug-prefix-map=a=b") { + defs = append(defs, "-fdebug-prefix-map="+b.WorkDir+"=/tmp/go-build") + } + if b.gccSupportsFlag(compiler, "-gno-record-gcc-switches") { + defs = append(defs, "-gno-record-gcc-switches") + } + return b.Shell(a).run(p.Dir, p.ImportPath, nil, compiler, "-Wall", "-g", + "-I", a.Objdir, "-I", inc, "-o", ofile, defs, "-c", cfile) +} + +// maybePIC adds -fPIC to the list of arguments if needed. +func (tools gccgoToolchain) maybePIC(args []string) []string { + switch cfg.BuildBuildmode { + case "c-shared", "shared", "plugin": + args = append(args, "-fPIC") + } + return args +} + +func gccgoPkgpath(p *load.Package) string { + if p.Internal.Build.IsCommand() && !p.Internal.ForceLibrary { + return "" + } + return p.ImportPath +} + +var gccgoToSymbolFuncOnce sync.Once +var gccgoToSymbolFunc func(string) string + +func (tools gccgoToolchain) gccgoCleanPkgpath(b *Builder, p *load.Package) string { + gccgoToSymbolFuncOnce.Do(func() { + tmpdir := b.WorkDir + if cfg.BuildN { + tmpdir = os.TempDir() + } + fn, err := pkgpath.ToSymbolFunc(tools.compiler(), tmpdir) + if err != nil { + fmt.Fprintf(os.Stderr, "cmd/go: %v\n", err) + base.SetExitStatus(2) + base.Exit() + } + gccgoToSymbolFunc = fn + }) + + return gccgoToSymbolFunc(gccgoPkgpath(p)) +} + +var ( + gccgoSupportsCgoIncompleteOnce sync.Once + gccgoSupportsCgoIncomplete bool +) + +const gccgoSupportsCgoIncompleteCode = ` +package p + +import "runtime/cgo" + +type I cgo.Incomplete +` + +// supportsCgoIncomplete reports whether the gccgo/GoLLVM compiler +// being used supports cgo.Incomplete, which was added in GCC 13. +// +// This takes an Action only for output reporting purposes. +// The result value is unrelated to the Action. +func (tools gccgoToolchain) supportsCgoIncomplete(b *Builder, a *Action) bool { + gccgoSupportsCgoIncompleteOnce.Do(func() { + sh := b.Shell(a) + + fail := func(err error) { + fmt.Fprintf(os.Stderr, "cmd/go: %v\n", err) + base.SetExitStatus(2) + base.Exit() + } + + tmpdir := b.WorkDir + if cfg.BuildN { + tmpdir = os.TempDir() + } + f, err := os.CreateTemp(tmpdir, "*_gccgo_cgoincomplete.go") + if err != nil { + fail(err) + } + fn := f.Name() + f.Close() + defer os.Remove(fn) + + if err := os.WriteFile(fn, []byte(gccgoSupportsCgoIncompleteCode), 0644); err != nil { + fail(err) + } + + on := strings.TrimSuffix(fn, ".go") + ".o" + if cfg.BuildN || cfg.BuildX { + sh.ShowCmd(tmpdir, "%s -c -o %s %s || true", tools.compiler(), on, fn) + // Since this function affects later builds, + // and only generates temporary files, + // we run the command even with -n. + } + cmd := exec.Command(tools.compiler(), "-c", "-o", on, fn) + cmd.Dir = tmpdir + var buf bytes.Buffer + cmd.Stdout = &buf + cmd.Stderr = &buf + err = cmd.Run() + gccgoSupportsCgoIncomplete = err == nil + if cfg.BuildN || cfg.BuildX { + // Show output. We always pass a nil err because errors are an + // expected outcome in this case. + desc := sh.fmtCmd(tmpdir, "%s -c -o %s %s", tools.compiler(), on, fn) + sh.reportCmd(desc, tmpdir, buf.Bytes(), nil) + } + }) + return gccgoSupportsCgoIncomplete +} diff --git a/src/cmd/go/internal/work/init.go b/src/cmd/go/internal/work/init.go new file mode 100644 index 0000000..7d0921f --- /dev/null +++ b/src/cmd/go/internal/work/init.go @@ -0,0 +1,424 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Build initialization (after flag parsing). + +package work + +import ( + "bytes" + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/modload" + "cmd/internal/quoted" + "fmt" + "internal/platform" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strconv" + "sync" +) + +var buildInitStarted = false + +func BuildInit() { + if buildInitStarted { + base.Fatalf("go: internal error: work.BuildInit called more than once") + } + buildInitStarted = true + base.AtExit(closeBuilders) + + modload.Init() + instrumentInit() + buildModeInit() + if err := fsys.Init(base.Cwd()); err != nil { + base.Fatal(err) + } + + // Make sure -pkgdir is absolute, because we run commands + // in different directories. + if cfg.BuildPkgdir != "" && !filepath.IsAbs(cfg.BuildPkgdir) { + p, err := filepath.Abs(cfg.BuildPkgdir) + if err != nil { + fmt.Fprintf(os.Stderr, "go: evaluating -pkgdir: %v\n", err) + base.SetExitStatus(2) + base.Exit() + } + cfg.BuildPkgdir = p + } + + if cfg.BuildP <= 0 { + base.Fatalf("go: -p must be a positive integer: %v\n", cfg.BuildP) + } + + // Make sure CC, CXX, and FC are absolute paths. + for _, key := range []string{"CC", "CXX", "FC"} { + value := cfg.Getenv(key) + args, err := quoted.Split(value) + if err != nil { + base.Fatalf("go: %s environment variable could not be parsed: %v", key, err) + } + if len(args) == 0 { + continue + } + path := args[0] + if !filepath.IsAbs(path) && path != filepath.Base(path) { + base.Fatalf("go: %s environment variable is relative; must be absolute path: %s\n", key, path) + } + } + + // Set covermode if not already set. + // Ensure that -race and -covermode are compatible. + if cfg.BuildCoverMode == "" { + cfg.BuildCoverMode = "set" + if cfg.BuildRace { + // Default coverage mode is atomic when -race is set. + cfg.BuildCoverMode = "atomic" + } + } + if cfg.BuildRace && cfg.BuildCoverMode != "atomic" { + base.Fatalf(`-covermode must be "atomic", not %q, when -race is enabled`, cfg.BuildCoverMode) + } +} + +// fuzzInstrumentFlags returns compiler flags that enable fuzzing instrumentation +// on supported platforms. +// +// On unsupported platforms, fuzzInstrumentFlags returns nil, meaning no +// instrumentation is added. 'go test -fuzz' still works without coverage, +// but it generates random inputs without guidance, so it's much less effective. +func fuzzInstrumentFlags() []string { + if !platform.FuzzInstrumented(cfg.Goos, cfg.Goarch) { + return nil + } + return []string{"-d=libfuzzer"} +} + +func instrumentInit() { + if !cfg.BuildRace && !cfg.BuildMSan && !cfg.BuildASan { + return + } + if cfg.BuildRace && cfg.BuildMSan { + fmt.Fprintf(os.Stderr, "go: may not use -race and -msan simultaneously\n") + base.SetExitStatus(2) + base.Exit() + } + if cfg.BuildRace && cfg.BuildASan { + fmt.Fprintf(os.Stderr, "go: may not use -race and -asan simultaneously\n") + base.SetExitStatus(2) + base.Exit() + } + if cfg.BuildMSan && cfg.BuildASan { + fmt.Fprintf(os.Stderr, "go: may not use -msan and -asan simultaneously\n") + base.SetExitStatus(2) + base.Exit() + } + if cfg.BuildMSan && !platform.MSanSupported(cfg.Goos, cfg.Goarch) { + fmt.Fprintf(os.Stderr, "-msan is not supported on %s/%s\n", cfg.Goos, cfg.Goarch) + base.SetExitStatus(2) + base.Exit() + } + if cfg.BuildRace && !platform.RaceDetectorSupported(cfg.Goos, cfg.Goarch) { + fmt.Fprintf(os.Stderr, "-race is not supported on %s/%s\n", cfg.Goos, cfg.Goarch) + base.SetExitStatus(2) + base.Exit() + } + if cfg.BuildASan && !platform.ASanSupported(cfg.Goos, cfg.Goarch) { + fmt.Fprintf(os.Stderr, "-asan is not supported on %s/%s\n", cfg.Goos, cfg.Goarch) + base.SetExitStatus(2) + base.Exit() + } + // The current implementation is only compatible with the ASan library from version + // v7 to v9 (See the description in src/runtime/asan/asan.go). Therefore, using the + // -asan option must use a compatible version of ASan library, which requires that + // the gcc version is not less than 7 and the clang version is not less than 9, + // otherwise a segmentation fault will occur. + if cfg.BuildASan { + if err := compilerRequiredAsanVersion(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + base.SetExitStatus(2) + base.Exit() + } + } + + mode := "race" + if cfg.BuildMSan { + mode = "msan" + // MSAN needs PIE on all platforms except linux/amd64. + // https://github.com/llvm/llvm-project/blob/llvmorg-13.0.1/clang/lib/Driver/SanitizerArgs.cpp#L621 + if cfg.BuildBuildmode == "default" && (cfg.Goos != "linux" || cfg.Goarch != "amd64") { + cfg.BuildBuildmode = "pie" + } + } + if cfg.BuildASan { + mode = "asan" + } + modeFlag := "-" + mode + + // Check that cgo is enabled. + // Note: On macOS, -race does not require cgo. -asan and -msan still do. + if !cfg.BuildContext.CgoEnabled && (cfg.Goos != "darwin" || cfg.BuildASan || cfg.BuildMSan) { + if runtime.GOOS != cfg.Goos || runtime.GOARCH != cfg.Goarch { + fmt.Fprintf(os.Stderr, "go: %s requires cgo\n", modeFlag) + } else { + fmt.Fprintf(os.Stderr, "go: %s requires cgo; enable cgo by setting CGO_ENABLED=1\n", modeFlag) + } + + base.SetExitStatus(2) + base.Exit() + } + forcedGcflags = append(forcedGcflags, modeFlag) + forcedLdflags = append(forcedLdflags, modeFlag) + + if cfg.BuildContext.InstallSuffix != "" { + cfg.BuildContext.InstallSuffix += "_" + } + cfg.BuildContext.InstallSuffix += mode + cfg.BuildContext.ToolTags = append(cfg.BuildContext.ToolTags, mode) +} + +func buildModeInit() { + gccgo := cfg.BuildToolchainName == "gccgo" + var codegenArg string + + // Configure the build mode first, then verify that it is supported. + // That way, if the flag is completely bogus we will prefer to error out with + // "-buildmode=%s not supported" instead of naming the specific platform. + + switch cfg.BuildBuildmode { + case "archive": + pkgsFilter = pkgsNotMain + case "c-archive": + pkgsFilter = oneMainPkg + if gccgo { + codegenArg = "-fPIC" + } else { + switch cfg.Goos { + case "darwin", "ios": + switch cfg.Goarch { + case "arm64": + codegenArg = "-shared" + } + + case "dragonfly", "freebsd", "illumos", "linux", "netbsd", "openbsd", "solaris": + // Use -shared so that the result is + // suitable for inclusion in a PIE or + // shared library. + codegenArg = "-shared" + } + } + cfg.ExeSuffix = ".a" + ldBuildmode = "c-archive" + case "c-shared": + pkgsFilter = oneMainPkg + if gccgo { + codegenArg = "-fPIC" + } else { + switch cfg.Goos { + case "linux", "android", "freebsd": + codegenArg = "-shared" + case "windows": + // Do not add usual .exe suffix to the .dll file. + cfg.ExeSuffix = "" + } + } + ldBuildmode = "c-shared" + case "default": + ldBuildmode = "exe" + if platform.DefaultPIE(cfg.Goos, cfg.Goarch, cfg.BuildRace) { + ldBuildmode = "pie" + if cfg.Goos != "windows" && !gccgo { + codegenArg = "-shared" + } + } + case "exe": + pkgsFilter = pkgsMain + ldBuildmode = "exe" + // Set the pkgsFilter to oneMainPkg if the user passed a specific binary output + // and is using buildmode=exe for a better error message. + // See issue #20017. + if cfg.BuildO != "" { + pkgsFilter = oneMainPkg + } + case "pie": + if cfg.BuildRace && !platform.DefaultPIE(cfg.Goos, cfg.Goarch, cfg.BuildRace) { + base.Fatalf("-buildmode=pie not supported when -race is enabled on %s/%s", cfg.Goos, cfg.Goarch) + } + if gccgo { + codegenArg = "-fPIE" + } else { + switch cfg.Goos { + case "aix", "windows": + default: + codegenArg = "-shared" + } + } + ldBuildmode = "pie" + case "shared": + pkgsFilter = pkgsNotMain + if gccgo { + codegenArg = "-fPIC" + } else { + codegenArg = "-dynlink" + } + if cfg.BuildO != "" { + base.Fatalf("-buildmode=shared and -o not supported together") + } + ldBuildmode = "shared" + case "plugin": + pkgsFilter = oneMainPkg + if gccgo { + codegenArg = "-fPIC" + } else { + codegenArg = "-dynlink" + } + cfg.ExeSuffix = ".so" + ldBuildmode = "plugin" + default: + base.Fatalf("buildmode=%s not supported", cfg.BuildBuildmode) + } + + if cfg.BuildBuildmode != "default" && !platform.BuildModeSupported(cfg.BuildToolchainName, cfg.BuildBuildmode, cfg.Goos, cfg.Goarch) { + base.Fatalf("-buildmode=%s not supported on %s/%s\n", cfg.BuildBuildmode, cfg.Goos, cfg.Goarch) + } + + if cfg.BuildLinkshared { + if !platform.BuildModeSupported(cfg.BuildToolchainName, "shared", cfg.Goos, cfg.Goarch) { + base.Fatalf("-linkshared not supported on %s/%s\n", cfg.Goos, cfg.Goarch) + } + if gccgo { + codegenArg = "-fPIC" + } else { + forcedAsmflags = append(forcedAsmflags, "-D=GOBUILDMODE_shared=1", + "-linkshared") + codegenArg = "-dynlink" + forcedGcflags = append(forcedGcflags, "-linkshared") + // TODO(mwhudson): remove -w when that gets fixed in linker. + forcedLdflags = append(forcedLdflags, "-linkshared", "-w") + } + } + if codegenArg != "" { + if gccgo { + forcedGccgoflags = append([]string{codegenArg}, forcedGccgoflags...) + } else { + forcedAsmflags = append([]string{codegenArg}, forcedAsmflags...) + forcedGcflags = append([]string{codegenArg}, forcedGcflags...) + } + // Don't alter InstallSuffix when modifying default codegen args. + if cfg.BuildBuildmode != "default" || cfg.BuildLinkshared { + if cfg.BuildContext.InstallSuffix != "" { + cfg.BuildContext.InstallSuffix += "_" + } + cfg.BuildContext.InstallSuffix += codegenArg[1:] + } + } + + switch cfg.BuildMod { + case "": + // Behavior will be determined automatically, as if no flag were passed. + case "readonly", "vendor", "mod": + if !cfg.ModulesEnabled && !base.InGOFLAGS("-mod") { + base.Fatalf("build flag -mod=%s only valid when using modules", cfg.BuildMod) + } + default: + base.Fatalf("-mod=%s not supported (can be '', 'mod', 'readonly', or 'vendor')", cfg.BuildMod) + } + if !cfg.ModulesEnabled { + if cfg.ModCacheRW && !base.InGOFLAGS("-modcacherw") { + base.Fatalf("build flag -modcacherw only valid when using modules") + } + if cfg.ModFile != "" && !base.InGOFLAGS("-mod") { + base.Fatalf("build flag -modfile only valid when using modules") + } + } +} + +type version struct { + name string + major, minor int +} + +var compiler struct { + sync.Once + version + err error +} + +// compilerVersion detects the version of $(go env CC). +// It returns a non-nil error if the compiler matches a known version schema but +// the version could not be parsed, or if $(go env CC) could not be determined. +func compilerVersion() (version, error) { + compiler.Once.Do(func() { + compiler.err = func() error { + compiler.name = "unknown" + cc := os.Getenv("CC") + out, err := exec.Command(cc, "--version").Output() + if err != nil { + // Compiler does not support "--version" flag: not Clang or GCC. + return err + } + + var match [][]byte + if bytes.HasPrefix(out, []byte("gcc")) { + compiler.name = "gcc" + out, err := exec.Command(cc, "-v").CombinedOutput() + if err != nil { + // gcc, but does not support gcc's "-v" flag?! + return err + } + gccRE := regexp.MustCompile(`gcc version (\d+)\.(\d+)`) + match = gccRE.FindSubmatch(out) + } else { + clangRE := regexp.MustCompile(`clang version (\d+)\.(\d+)`) + if match = clangRE.FindSubmatch(out); len(match) > 0 { + compiler.name = "clang" + } + } + + if len(match) < 3 { + return nil // "unknown" + } + if compiler.major, err = strconv.Atoi(string(match[1])); err != nil { + return err + } + if compiler.minor, err = strconv.Atoi(string(match[2])); err != nil { + return err + } + return nil + }() + }) + return compiler.version, compiler.err +} + +// compilerRequiredAsanVersion is a copy of the function defined in +// cmd/cgo/internal/testsanitizers/cc_test.go +// compilerRequiredAsanVersion reports whether the compiler is the version +// required by Asan. +func compilerRequiredAsanVersion() error { + compiler, err := compilerVersion() + if err != nil { + return fmt.Errorf("-asan: the version of $(go env CC) could not be parsed") + } + + switch compiler.name { + case "gcc": + if runtime.GOARCH == "ppc64le" && compiler.major < 9 { + return fmt.Errorf("-asan is not supported with %s compiler %d.%d\n", compiler.name, compiler.major, compiler.minor) + } + if compiler.major < 7 { + return fmt.Errorf("-asan is not supported with %s compiler %d.%d\n", compiler.name, compiler.major, compiler.minor) + } + case "clang": + if compiler.major < 9 { + return fmt.Errorf("-asan is not supported with %s compiler %d.%d\n", compiler.name, compiler.major, compiler.minor) + } + default: + return fmt.Errorf("-asan: C compiler is not gcc or clang") + } + return nil +} diff --git a/src/cmd/go/internal/work/security.go b/src/cmd/go/internal/work/security.go new file mode 100644 index 0000000..88504be --- /dev/null +++ b/src/cmd/go/internal/work/security.go @@ -0,0 +1,338 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Checking of compiler and linker flags. +// We must avoid flags like -fplugin=, which can allow +// arbitrary code execution during the build. +// Do not make changes here without carefully +// considering the implications. +// (That's why the code is isolated in a file named security.go.) +// +// Note that -Wl,foo means split foo on commas and pass to +// the linker, so that -Wl,-foo,bar means pass -foo bar to +// the linker. Similarly -Wa,foo for the assembler and so on. +// If any of these are permitted, the wildcard portion must +// disallow commas. +// +// Note also that GNU binutils accept any argument @foo +// as meaning "read more flags from the file foo", so we must +// guard against any command-line argument beginning with @, +// even things like "-I @foo". +// We use load.SafeArg (which is even more conservative) +// to reject these. +// +// Even worse, gcc -I@foo (one arg) turns into cc1 -I @foo (two args), +// so although gcc doesn't expand the @foo, cc1 will. +// So out of paranoia, we reject @ at the beginning of every +// flag argument that might be split into its own argument. + +package work + +import ( + "fmt" + "internal/lazyregexp" + "regexp" + "strings" + + "cmd/go/internal/cfg" + "cmd/go/internal/load" +) + +var re = lazyregexp.New + +var validCompilerFlags = []*lazyregexp.Regexp{ + re(`-D([A-Za-z_][A-Za-z0-9_]*)(=[^@\-]*)?`), + re(`-U([A-Za-z_][A-Za-z0-9_]*)`), + re(`-F([^@\-].*)`), + re(`-I([^@\-].*)`), + re(`-O`), + re(`-O([^@\-].*)`), + re(`-W`), + re(`-W([^@,]+)`), // -Wall but not -Wa,-foo. + re(`-Wa,-mbig-obj`), + re(`-Wp,-D([A-Za-z_][A-Za-z0-9_]*)(=[^@,\-]*)?`), + re(`-Wp,-U([A-Za-z_][A-Za-z0-9_]*)`), + re(`-ansi`), + re(`-f(no-)?asynchronous-unwind-tables`), + re(`-f(no-)?blocks`), + re(`-f(no-)builtin-[a-zA-Z0-9_]*`), + re(`-f(no-)?common`), + re(`-f(no-)?constant-cfstrings`), + re(`-fdebug-prefix-map=([^@]+)=([^@]+)`), + re(`-fdiagnostics-show-note-include-stack`), + re(`-ffile-prefix-map=([^@]+)=([^@]+)`), + re(`-fno-canonical-system-headers`), + re(`-f(no-)?eliminate-unused-debug-types`), + re(`-f(no-)?exceptions`), + re(`-f(no-)?fast-math`), + re(`-f(no-)?inline-functions`), + re(`-finput-charset=([^@\-].*)`), + re(`-f(no-)?fat-lto-objects`), + re(`-f(no-)?keep-inline-dllexport`), + re(`-f(no-)?lto`), + re(`-fmacro-backtrace-limit=(.+)`), + re(`-fmessage-length=(.+)`), + re(`-f(no-)?modules`), + re(`-f(no-)?objc-arc`), + re(`-f(no-)?objc-nonfragile-abi`), + re(`-f(no-)?objc-legacy-dispatch`), + re(`-f(no-)?omit-frame-pointer`), + re(`-f(no-)?openmp(-simd)?`), + re(`-f(no-)?permissive`), + re(`-f(no-)?(pic|PIC|pie|PIE)`), + re(`-f(no-)?plt`), + re(`-f(no-)?rtti`), + re(`-f(no-)?split-stack`), + re(`-f(no-)?stack-(.+)`), + re(`-f(no-)?strict-aliasing`), + re(`-f(un)signed-char`), + re(`-f(no-)?use-linker-plugin`), // safe if -B is not used; we don't permit -B + re(`-f(no-)?visibility-inlines-hidden`), + re(`-fsanitize=(.+)`), + re(`-ftemplate-depth-(.+)`), + re(`-fvisibility=(.+)`), + re(`-g([^@\-].*)?`), + re(`-m32`), + re(`-m64`), + re(`-m(abi|arch|cpu|fpu|tune)=([^@\-].*)`), + re(`-m(no-)?v?aes`), + re(`-marm`), + re(`-m(no-)?avx[0-9a-z]*`), + re(`-mfloat-abi=([^@\-].*)`), + re(`-mfpmath=[0-9a-z,+]*`), + re(`-m(no-)?avx[0-9a-z.]*`), + re(`-m(no-)?ms-bitfields`), + re(`-m(no-)?stack-(.+)`), + re(`-mmacosx-(.+)`), + re(`-mios-simulator-version-min=(.+)`), + re(`-miphoneos-version-min=(.+)`), + re(`-mtvos-simulator-version-min=(.+)`), + re(`-mtvos-version-min=(.+)`), + re(`-mwatchos-simulator-version-min=(.+)`), + re(`-mwatchos-version-min=(.+)`), + re(`-mnop-fun-dllimport`), + re(`-m(no-)?sse[0-9.]*`), + re(`-m(no-)?ssse3`), + re(`-mthumb(-interwork)?`), + re(`-mthreads`), + re(`-mwindows`), + re(`-no-canonical-prefixes`), + re(`--param=ssp-buffer-size=[0-9]*`), + re(`-pedantic(-errors)?`), + re(`-pipe`), + re(`-pthread`), + re(`-?-std=([^@\-].*)`), + re(`-?-stdlib=([^@\-].*)`), + re(`--sysroot=([^@\-].*)`), + re(`-w`), + re(`-x([^@\-].*)`), + re(`-v`), +} + +var validCompilerFlagsWithNextArg = []string{ + "-arch", + "-D", + "-U", + "-I", + "-F", + "-framework", + "-include", + "-isysroot", + "-isystem", + "--sysroot", + "-target", + "-x", +} + +var validLinkerFlags = []*lazyregexp.Regexp{ + re(`-F([^@\-].*)`), + re(`-l([^@\-].*)`), + re(`-L([^@\-].*)`), + re(`-O`), + re(`-O([^@\-].*)`), + re(`-f(no-)?(pic|PIC|pie|PIE)`), + re(`-f(no-)?openmp(-simd)?`), + re(`-fsanitize=([^@\-].*)`), + re(`-flat_namespace`), + re(`-g([^@\-].*)?`), + re(`-headerpad_max_install_names`), + re(`-m(abi|arch|cpu|fpu|tune)=([^@\-].*)`), + re(`-mfloat-abi=([^@\-].*)`), + re(`-mmacosx-(.+)`), + re(`-mios-simulator-version-min=(.+)`), + re(`-miphoneos-version-min=(.+)`), + re(`-mthreads`), + re(`-mwindows`), + re(`-(pic|PIC|pie|PIE)`), + re(`-pthread`), + re(`-rdynamic`), + re(`-shared`), + re(`-?-static([-a-z0-9+]*)`), + re(`-?-stdlib=([^@\-].*)`), + re(`-v`), + + // Note that any wildcards in -Wl need to exclude comma, + // since -Wl splits its argument at commas and passes + // them all to the linker uninterpreted. Allowing comma + // in a wildcard would allow tunneling arbitrary additional + // linker arguments through one of these. + re(`-Wl,--(no-)?allow-multiple-definition`), + re(`-Wl,--(no-)?allow-shlib-undefined`), + re(`-Wl,--(no-)?as-needed`), + re(`-Wl,-Bdynamic`), + re(`-Wl,-berok`), + re(`-Wl,-Bstatic`), + re(`-Wl,-Bsymbolic-functions`), + re(`-Wl,-O[0-9]+`), + re(`-Wl,-d[ny]`), + re(`-Wl,--disable-new-dtags`), + re(`-Wl,-e[=,][a-zA-Z0-9]+`), + re(`-Wl,--enable-new-dtags`), + re(`-Wl,--end-group`), + re(`-Wl,--(no-)?export-dynamic`), + re(`-Wl,-E`), + re(`-Wl,-framework,[^,@\-][^,]+`), + re(`-Wl,--hash-style=(sysv|gnu|both)`), + re(`-Wl,-headerpad_max_install_names`), + re(`-Wl,--no-undefined`), + re(`-Wl,-R,?([^@\-,][^,@]*$)`), + re(`-Wl,--just-symbols[=,]([^,@\-][^,@]+)`), + re(`-Wl,-rpath(-link)?[=,]([^,@\-][^,]+)`), + re(`-Wl,-s`), + re(`-Wl,-search_paths_first`), + re(`-Wl,-sectcreate,([^,@\-][^,]+),([^,@\-][^,]+),([^,@\-][^,]+)`), + re(`-Wl,--start-group`), + re(`-Wl,-?-static`), + re(`-Wl,-?-subsystem,(native|windows|console|posix|xbox)`), + re(`-Wl,-syslibroot[=,]([^,@\-][^,]+)`), + re(`-Wl,-undefined[=,]([^,@\-][^,]+)`), + re(`-Wl,-?-unresolved-symbols=[^,]+`), + re(`-Wl,--(no-)?warn-([^,]+)`), + re(`-Wl,-?-wrap[=,][^,@\-][^,]*`), + re(`-Wl,-z,(no)?execstack`), + re(`-Wl,-z,relro`), + + re(`[a-zA-Z0-9_/].*\.(a|o|obj|dll|dylib|so|tbd)`), // direct linker inputs: x.o or libfoo.so (but not -foo.o or @foo.o) + re(`\./.*\.(a|o|obj|dll|dylib|so|tbd)`), +} + +var validLinkerFlagsWithNextArg = []string{ + "-arch", + "-F", + "-l", + "-L", + "-framework", + "-isysroot", + "--sysroot", + "-target", + "-Wl,-framework", + "-Wl,-rpath", + "-Wl,-R", + "-Wl,--just-symbols", + "-Wl,-undefined", +} + +func checkCompilerFlags(name, source string, list []string) error { + checkOverrides := true + return checkFlags(name, source, list, validCompilerFlags, validCompilerFlagsWithNextArg, checkOverrides) +} + +func checkLinkerFlags(name, source string, list []string) error { + checkOverrides := true + return checkFlags(name, source, list, validLinkerFlags, validLinkerFlagsWithNextArg, checkOverrides) +} + +// checkCompilerFlagsForInternalLink returns an error if 'list' +// contains a flag or flags that may not be fully supported by +// internal linking (meaning that we should punt the link to the +// external linker). +func checkCompilerFlagsForInternalLink(name, source string, list []string) error { + checkOverrides := false + if err := checkFlags(name, source, list, validCompilerFlags, validCompilerFlagsWithNextArg, checkOverrides); err != nil { + return err + } + // Currently the only flag on the allow list that causes problems + // for the linker is "-flto"; check for it manually here. + for _, fl := range list { + if strings.HasPrefix(fl, "-flto") { + return fmt.Errorf("flag %q triggers external linking", fl) + } + } + return nil +} + +func checkFlags(name, source string, list []string, valid []*lazyregexp.Regexp, validNext []string, checkOverrides bool) error { + // Let users override rules with $CGO_CFLAGS_ALLOW, $CGO_CFLAGS_DISALLOW, etc. + var ( + allow *regexp.Regexp + disallow *regexp.Regexp + ) + if checkOverrides { + if env := cfg.Getenv("CGO_" + name + "_ALLOW"); env != "" { + r, err := regexp.Compile(env) + if err != nil { + return fmt.Errorf("parsing $CGO_%s_ALLOW: %v", name, err) + } + allow = r + } + if env := cfg.Getenv("CGO_" + name + "_DISALLOW"); env != "" { + r, err := regexp.Compile(env) + if err != nil { + return fmt.Errorf("parsing $CGO_%s_DISALLOW: %v", name, err) + } + disallow = r + } + } + +Args: + for i := 0; i < len(list); i++ { + arg := list[i] + if disallow != nil && disallow.FindString(arg) == arg { + goto Bad + } + if allow != nil && allow.FindString(arg) == arg { + continue Args + } + for _, re := range valid { + if re.FindString(arg) == arg { // must be complete match + continue Args + } + } + for _, x := range validNext { + if arg == x { + if i+1 < len(list) && load.SafeArg(list[i+1]) { + i++ + continue Args + } + + // Permit -Wl,-framework -Wl,name. + if i+1 < len(list) && + strings.HasPrefix(arg, "-Wl,") && + strings.HasPrefix(list[i+1], "-Wl,") && + load.SafeArg(list[i+1][4:]) && + !strings.Contains(list[i+1][4:], ",") { + i++ + continue Args + } + + // Permit -I= /path, -I $SYSROOT. + if i+1 < len(list) && arg == "-I" { + if (strings.HasPrefix(list[i+1], "=") || strings.HasPrefix(list[i+1], "$SYSROOT")) && + load.SafeArg(list[i+1][1:]) { + i++ + continue Args + } + } + + if i+1 < len(list) { + return fmt.Errorf("invalid flag in %s: %s %s (see https://golang.org/s/invalidflag)", source, arg, list[i+1]) + } + return fmt.Errorf("invalid flag in %s: %s without argument (see https://golang.org/s/invalidflag)", source, arg) + } + } + Bad: + return fmt.Errorf("invalid flag in %s: %s", source, arg) + } + return nil +} diff --git a/src/cmd/go/internal/work/security_test.go b/src/cmd/go/internal/work/security_test.go new file mode 100644 index 0000000..c05ba7b --- /dev/null +++ b/src/cmd/go/internal/work/security_test.go @@ -0,0 +1,318 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "os" + "strings" + "testing" +) + +var goodCompilerFlags = [][]string{ + {"-DFOO"}, + {"-Dfoo=bar"}, + {"-Ufoo"}, + {"-Ufoo1"}, + {"-F/Qt"}, + {"-F", "/Qt"}, + {"-I/"}, + {"-I/etc/passwd"}, + {"-I."}, + {"-O"}, + {"-O2"}, + {"-Osmall"}, + {"-W"}, + {"-Wall"}, + {"-Wp,-Dfoo=bar"}, + {"-Wp,-Ufoo"}, + {"-Wp,-Dfoo1"}, + {"-Wp,-Ufoo1"}, + {"-flto"}, + {"-fobjc-arc"}, + {"-fno-objc-arc"}, + {"-fomit-frame-pointer"}, + {"-fno-omit-frame-pointer"}, + {"-fpic"}, + {"-fno-pic"}, + {"-fPIC"}, + {"-fno-PIC"}, + {"-fpie"}, + {"-fno-pie"}, + {"-fPIE"}, + {"-fno-PIE"}, + {"-fsplit-stack"}, + {"-fno-split-stack"}, + {"-fstack-xxx"}, + {"-fno-stack-xxx"}, + {"-fsanitize=hands"}, + {"-g"}, + {"-ggdb"}, + {"-march=souza"}, + {"-mcpu=123"}, + {"-mfpu=123"}, + {"-mtune=happybirthday"}, + {"-mstack-overflow"}, + {"-mno-stack-overflow"}, + {"-mmacosx-version"}, + {"-mnop-fun-dllimport"}, + {"-pthread"}, + {"-std=c99"}, + {"-xc"}, + {"-D", "FOO"}, + {"-D", "foo=bar"}, + {"-I", "."}, + {"-I", "/etc/passwd"}, + {"-I", "世界"}, + {"-I", "=/usr/include/libxml2"}, + {"-I", "dir"}, + {"-I", "$SYSROOT/dir"}, + {"-isystem", "/usr/include/mozjs-68"}, + {"-include", "/usr/include/mozjs-68/RequiredDefines.h"}, + {"-framework", "Chocolate"}, + {"-x", "c"}, + {"-v"}, +} + +var badCompilerFlags = [][]string{ + {"-D@X"}, + {"-D-X"}, + {"-Ufoo=bar"}, + {"-F@dir"}, + {"-F-dir"}, + {"-I@dir"}, + {"-I-dir"}, + {"-O@1"}, + {"-Wa,-foo"}, + {"-W@foo"}, + {"-Wp,-DX,-D@X"}, + {"-Wp,-UX,-U@X"}, + {"-g@gdb"}, + {"-g-gdb"}, + {"-march=@dawn"}, + {"-march=-dawn"}, + {"-std=@c99"}, + {"-std=-c99"}, + {"-x@c"}, + {"-x-c"}, + {"-D", "@foo"}, + {"-D", "-foo"}, + {"-I", "@foo"}, + {"-I", "-foo"}, + {"-I", "=@obj"}, + {"-include", "@foo"}, + {"-framework", "-Caffeine"}, + {"-framework", "@Home"}, + {"-x", "--c"}, + {"-x", "@obj"}, +} + +func TestCheckCompilerFlags(t *testing.T) { + for _, f := range goodCompilerFlags { + if err := checkCompilerFlags("test", "test", f); err != nil { + t.Errorf("unexpected error for %q: %v", f, err) + } + } + for _, f := range badCompilerFlags { + if err := checkCompilerFlags("test", "test", f); err == nil { + t.Errorf("missing error for %q", f) + } + } +} + +var goodLinkerFlags = [][]string{ + {"-Fbar"}, + {"-lbar"}, + {"-Lbar"}, + {"-fpic"}, + {"-fno-pic"}, + {"-fPIC"}, + {"-fno-PIC"}, + {"-fpie"}, + {"-fno-pie"}, + {"-fPIE"}, + {"-fno-PIE"}, + {"-fsanitize=hands"}, + {"-g"}, + {"-ggdb"}, + {"-march=souza"}, + {"-mcpu=123"}, + {"-mfpu=123"}, + {"-mtune=happybirthday"}, + {"-pic"}, + {"-pthread"}, + {"-Wl,--hash-style=both"}, + {"-Wl,-rpath,foo"}, + {"-Wl,-rpath,$ORIGIN/foo"}, + {"-Wl,-R", "/foo"}, + {"-Wl,-R", "foo"}, + {"-Wl,-R,foo"}, + {"-Wl,--just-symbols=foo"}, + {"-Wl,--just-symbols,foo"}, + {"-Wl,--warn-error"}, + {"-Wl,--no-warn-error"}, + {"foo.so"}, + {"_世界.dll"}, + {"./x.o"}, + {"libcgosotest.dylib"}, + {"-F", "framework"}, + {"-l", "."}, + {"-l", "/etc/passwd"}, + {"-l", "世界"}, + {"-L", "framework"}, + {"-framework", "Chocolate"}, + {"-v"}, + {"-Wl,-sectcreate,__TEXT,__info_plist,${SRCDIR}/Info.plist"}, + {"-Wl,-framework", "-Wl,Chocolate"}, + {"-Wl,-framework,Chocolate"}, + {"-Wl,-unresolved-symbols=ignore-all"}, + {"libcgotbdtest.tbd"}, + {"./libcgotbdtest.tbd"}, +} + +var badLinkerFlags = [][]string{ + {"-DFOO"}, + {"-Dfoo=bar"}, + {"-W"}, + {"-Wall"}, + {"-fobjc-arc"}, + {"-fno-objc-arc"}, + {"-fomit-frame-pointer"}, + {"-fno-omit-frame-pointer"}, + {"-fsplit-stack"}, + {"-fno-split-stack"}, + {"-fstack-xxx"}, + {"-fno-stack-xxx"}, + {"-mstack-overflow"}, + {"-mno-stack-overflow"}, + {"-mnop-fun-dllimport"}, + {"-std=c99"}, + {"-xc"}, + {"-D", "FOO"}, + {"-D", "foo=bar"}, + {"-I", "FOO"}, + {"-L", "@foo"}, + {"-L", "-foo"}, + {"-x", "c"}, + {"-D@X"}, + {"-D-X"}, + {"-I@dir"}, + {"-I-dir"}, + {"-O@1"}, + {"-Wa,-foo"}, + {"-W@foo"}, + {"-g@gdb"}, + {"-g-gdb"}, + {"-march=@dawn"}, + {"-march=-dawn"}, + {"-std=@c99"}, + {"-std=-c99"}, + {"-x@c"}, + {"-x-c"}, + {"-D", "@foo"}, + {"-D", "-foo"}, + {"-I", "@foo"}, + {"-I", "-foo"}, + {"-l", "@foo"}, + {"-l", "-foo"}, + {"-framework", "-Caffeine"}, + {"-framework", "@Home"}, + {"-Wl,-framework,-Caffeine"}, + {"-Wl,-framework", "-Wl,@Home"}, + {"-Wl,-framework", "@Home"}, + {"-Wl,-framework,Chocolate,@Home"}, + {"-Wl,--hash-style=foo"}, + {"-x", "--c"}, + {"-x", "@obj"}, + {"-Wl,-rpath,@foo"}, + {"-Wl,-R,foo,bar"}, + {"-Wl,-R,@foo"}, + {"-Wl,--just-symbols,@foo"}, + {"../x.o"}, + {"-Wl,-R,"}, + {"-Wl,-O"}, + {"-Wl,-e="}, + {"-Wl,-e,"}, + {"-Wl,-R,-flag"}, +} + +func TestCheckLinkerFlags(t *testing.T) { + for _, f := range goodLinkerFlags { + if err := checkLinkerFlags("test", "test", f); err != nil { + t.Errorf("unexpected error for %q: %v", f, err) + } + } + for _, f := range badLinkerFlags { + if err := checkLinkerFlags("test", "test", f); err == nil { + t.Errorf("missing error for %q", f) + } + } +} + +func TestCheckFlagAllowDisallow(t *testing.T) { + if err := checkCompilerFlags("TEST", "test", []string{"-disallow"}); err == nil { + t.Fatalf("missing error for -disallow") + } + os.Setenv("CGO_TEST_ALLOW", "-disallo") + if err := checkCompilerFlags("TEST", "test", []string{"-disallow"}); err == nil { + t.Fatalf("missing error for -disallow with CGO_TEST_ALLOW=-disallo") + } + os.Setenv("CGO_TEST_ALLOW", "-disallow") + if err := checkCompilerFlags("TEST", "test", []string{"-disallow"}); err != nil { + t.Fatalf("unexpected error for -disallow with CGO_TEST_ALLOW=-disallow: %v", err) + } + os.Unsetenv("CGO_TEST_ALLOW") + + if err := checkCompilerFlags("TEST", "test", []string{"-Wall"}); err != nil { + t.Fatalf("unexpected error for -Wall: %v", err) + } + os.Setenv("CGO_TEST_DISALLOW", "-Wall") + if err := checkCompilerFlags("TEST", "test", []string{"-Wall"}); err == nil { + t.Fatalf("missing error for -Wall with CGO_TEST_DISALLOW=-Wall") + } + os.Setenv("CGO_TEST_ALLOW", "-Wall") // disallow wins + if err := checkCompilerFlags("TEST", "test", []string{"-Wall"}); err == nil { + t.Fatalf("missing error for -Wall with CGO_TEST_DISALLOW=-Wall and CGO_TEST_ALLOW=-Wall") + } + + os.Setenv("CGO_TEST_ALLOW", "-fplugin.*") + os.Setenv("CGO_TEST_DISALLOW", "-fplugin=lint.so") + if err := checkCompilerFlags("TEST", "test", []string{"-fplugin=faster.so"}); err != nil { + t.Fatalf("unexpected error for -fplugin=faster.so: %v", err) + } + if err := checkCompilerFlags("TEST", "test", []string{"-fplugin=lint.so"}); err == nil { + t.Fatalf("missing error for -fplugin=lint.so: %v", err) + } +} + +func TestCheckCompilerFlagsForInternalLink(t *testing.T) { + // Any "bad" compiler flag should trigger external linking. + for _, f := range badCompilerFlags { + if err := checkCompilerFlagsForInternalLink("test", "test", f); err == nil { + t.Errorf("missing error for %q", f) + } + } + + // All "good" compiler flags should not trigger external linking, + // except for anything that begins with "-flto". + for _, f := range goodCompilerFlags { + foundLTO := false + for _, s := range f { + if strings.Contains(s, "-flto") { + foundLTO = true + } + } + if err := checkCompilerFlagsForInternalLink("test", "test", f); err != nil { + // expect error for -flto + if !foundLTO { + t.Errorf("unexpected error for %q: %v", f, err) + } + } else { + // expect no error for everything else + if foundLTO { + t.Errorf("missing error for %q: %v", f, err) + } + } + } +} diff --git a/src/cmd/go/internal/work/shell.go b/src/cmd/go/internal/work/shell.go new file mode 100644 index 0000000..6089170 --- /dev/null +++ b/src/cmd/go/internal/work/shell.go @@ -0,0 +1,678 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "bytes" + "cmd/go/internal/base" + "cmd/go/internal/cache" + "cmd/go/internal/cfg" + "cmd/go/internal/load" + "cmd/go/internal/par" + "cmd/go/internal/str" + "errors" + "fmt" + "internal/lazyregexp" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "time" +) + +// A Shell runs shell commands and performs shell-like file system operations. +// +// Shell tracks context related to running commands, and form a tree much like +// context.Context. +type Shell struct { + action *Action // nil for the root shell + *shellShared // per-Builder state shared across Shells +} + +// shellShared is Shell state shared across all Shells derived from a single +// root shell (generally a single Builder). +type shellShared struct { + workDir string // $WORK, immutable + + printLock sync.Mutex + printFunc func(args ...any) (int, error) + scriptDir string // current directory in printed script + + mkdirCache par.Cache[string, error] // a cache of created directories +} + +// NewShell returns a new Shell. +// +// Shell will internally serialize calls to the print function. +// If print is nil, it defaults to printing to stderr. +func NewShell(workDir string, print func(a ...any) (int, error)) *Shell { + if print == nil { + print = func(a ...any) (int, error) { + return fmt.Fprint(os.Stderr, a...) + } + } + shared := &shellShared{ + workDir: workDir, + printFunc: print, + } + return &Shell{shellShared: shared} +} + +// Print emits a to this Shell's output stream, formatting it like fmt.Print. +// It is safe to call concurrently. +func (sh *Shell) Print(a ...any) { + sh.printLock.Lock() + defer sh.printLock.Unlock() + sh.printFunc(a...) +} + +func (sh *Shell) printLocked(a ...any) { + sh.printFunc(a...) +} + +// WithAction returns a Shell identical to sh, but bound to Action a. +func (sh *Shell) WithAction(a *Action) *Shell { + sh2 := *sh + sh2.action = a + return &sh2 +} + +// Shell returns a shell for running commands on behalf of Action a. +func (b *Builder) Shell(a *Action) *Shell { + if a == nil { + // The root shell has a nil Action. The point of this method is to + // create a Shell bound to an Action, so disallow nil Actions here. + panic("nil Action") + } + if a.sh == nil { + a.sh = b.backgroundSh.WithAction(a) + } + return a.sh +} + +// BackgroundShell returns a Builder-wide Shell that's not bound to any Action. +// Try not to use this unless there's really no sensible Action available. +func (b *Builder) BackgroundShell() *Shell { + return b.backgroundSh +} + +// moveOrCopyFile is like 'mv src dst' or 'cp src dst'. +func (sh *Shell) moveOrCopyFile(dst, src string, perm fs.FileMode, force bool) error { + if cfg.BuildN { + sh.ShowCmd("", "mv %s %s", src, dst) + return nil + } + + // If we can update the mode and rename to the dst, do it. + // Otherwise fall back to standard copy. + + // If the source is in the build cache, we need to copy it. + if strings.HasPrefix(src, cache.DefaultDir()) { + return sh.CopyFile(dst, src, perm, force) + } + + // On Windows, always copy the file, so that we respect the NTFS + // permissions of the parent folder. https://golang.org/issue/22343. + // What matters here is not cfg.Goos (the system we are building + // for) but runtime.GOOS (the system we are building on). + if runtime.GOOS == "windows" { + return sh.CopyFile(dst, src, perm, force) + } + + // If the destination directory has the group sticky bit set, + // we have to copy the file to retain the correct permissions. + // https://golang.org/issue/18878 + if fi, err := os.Stat(filepath.Dir(dst)); err == nil { + if fi.IsDir() && (fi.Mode()&fs.ModeSetgid) != 0 { + return sh.CopyFile(dst, src, perm, force) + } + } + + // The perm argument is meant to be adjusted according to umask, + // but we don't know what the umask is. + // Create a dummy file to find out. + // This avoids build tags and works even on systems like Plan 9 + // where the file mask computation incorporates other information. + mode := perm + f, err := os.OpenFile(filepath.Clean(dst)+"-go-tmp-umask", os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm) + if err == nil { + fi, err := f.Stat() + if err == nil { + mode = fi.Mode() & 0777 + } + name := f.Name() + f.Close() + os.Remove(name) + } + + if err := os.Chmod(src, mode); err == nil { + if err := os.Rename(src, dst); err == nil { + if cfg.BuildX { + sh.ShowCmd("", "mv %s %s", src, dst) + } + return nil + } + } + + return sh.CopyFile(dst, src, perm, force) +} + +// copyFile is like 'cp src dst'. +func (sh *Shell) CopyFile(dst, src string, perm fs.FileMode, force bool) error { + if cfg.BuildN || cfg.BuildX { + sh.ShowCmd("", "cp %s %s", src, dst) + if cfg.BuildN { + return nil + } + } + + sf, err := os.Open(src) + if err != nil { + return err + } + defer sf.Close() + + // Be careful about removing/overwriting dst. + // Do not remove/overwrite if dst exists and is a directory + // or a non-empty non-object file. + if fi, err := os.Stat(dst); err == nil { + if fi.IsDir() { + return fmt.Errorf("build output %q already exists and is a directory", dst) + } + if !force && fi.Mode().IsRegular() && fi.Size() != 0 && !isObject(dst) { + return fmt.Errorf("build output %q already exists and is not an object file", dst) + } + } + + // On Windows, remove lingering ~ file from last attempt. + if runtime.GOOS == "windows" { + if _, err := os.Stat(dst + "~"); err == nil { + os.Remove(dst + "~") + } + } + + mayberemovefile(dst) + df, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil && runtime.GOOS == "windows" { + // Windows does not allow deletion of a binary file + // while it is executing. Try to move it out of the way. + // If the move fails, which is likely, we'll try again the + // next time we do an install of this binary. + if err := os.Rename(dst, dst+"~"); err == nil { + os.Remove(dst + "~") + } + df, err = os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + } + if err != nil { + return fmt.Errorf("copying %s: %w", src, err) // err should already refer to dst + } + + _, err = io.Copy(df, sf) + df.Close() + if err != nil { + mayberemovefile(dst) + return fmt.Errorf("copying %s to %s: %v", src, dst, err) + } + return nil +} + +// mayberemovefile removes a file only if it is a regular file +// When running as a user with sufficient privileges, we may delete +// even device files, for example, which is not intended. +func mayberemovefile(s string) { + if fi, err := os.Lstat(s); err == nil && !fi.Mode().IsRegular() { + return + } + os.Remove(s) +} + +// writeFile writes the text to file. +func (sh *Shell) writeFile(file string, text []byte) error { + if cfg.BuildN || cfg.BuildX { + switch { + case len(text) == 0: + sh.ShowCmd("", "echo -n > %s # internal", file) + case bytes.IndexByte(text, '\n') == len(text)-1: + // One line. Use a simpler "echo" command. + sh.ShowCmd("", "echo '%s' > %s # internal", bytes.TrimSuffix(text, []byte("\n")), file) + default: + // Use the most general form. + sh.ShowCmd("", "cat >%s << 'EOF' # internal\n%sEOF", file, text) + } + } + if cfg.BuildN { + return nil + } + return os.WriteFile(file, text, 0666) +} + +// Mkdir makes the named directory. +func (sh *Shell) Mkdir(dir string) error { + // Make Mkdir(a.Objdir) a no-op instead of an error when a.Objdir == "". + if dir == "" { + return nil + } + + // We can be a little aggressive about being + // sure directories exist. Skip repeated calls. + return sh.mkdirCache.Do(dir, func() error { + if cfg.BuildN || cfg.BuildX { + sh.ShowCmd("", "mkdir -p %s", dir) + if cfg.BuildN { + return nil + } + } + + return os.MkdirAll(dir, 0777) + }) +} + +// RemoveAll is like 'rm -rf'. It attempts to remove all paths even if there's +// an error, and returns the first error. +func (sh *Shell) RemoveAll(paths ...string) error { + if cfg.BuildN || cfg.BuildX { + // Don't say we are removing the directory if we never created it. + show := func() bool { + for _, path := range paths { + if _, ok := sh.mkdirCache.Get(path); ok { + return true + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + return true + } + } + return false + } + if show() { + sh.ShowCmd("", "rm -rf %s", strings.Join(paths, " ")) + } + } + if cfg.BuildN { + return nil + } + + var err error + for _, path := range paths { + if err2 := os.RemoveAll(path); err2 != nil && err == nil { + err = err2 + } + } + return err +} + +// Symlink creates a symlink newname -> oldname. +func (sh *Shell) Symlink(oldname, newname string) error { + // It's not an error to try to recreate an existing symlink. + if link, err := os.Readlink(newname); err == nil && link == oldname { + return nil + } + + if cfg.BuildN || cfg.BuildX { + sh.ShowCmd("", "ln -s %s %s", oldname, newname) + if cfg.BuildN { + return nil + } + } + return os.Symlink(oldname, newname) +} + +// fmtCmd formats a command in the manner of fmt.Sprintf but also: +// +// fmtCmd replaces the value of b.WorkDir with $WORK. +func (sh *Shell) fmtCmd(dir string, format string, args ...any) string { + cmd := fmt.Sprintf(format, args...) + if sh.workDir != "" && !strings.HasPrefix(cmd, "cat ") { + cmd = strings.ReplaceAll(cmd, sh.workDir, "$WORK") + escaped := strconv.Quote(sh.workDir) + escaped = escaped[1 : len(escaped)-1] // strip quote characters + if escaped != sh.workDir { + cmd = strings.ReplaceAll(cmd, escaped, "$WORK") + } + } + return cmd +} + +// ShowCmd prints the given command to standard output +// for the implementation of -n or -x. +// +// ShowCmd also replaces the name of the current script directory with dot (.) +// but only when it is at the beginning of a space-separated token. +// +// If dir is not "" or "/" and not the current script directory, ShowCmd first +// prints a "cd" command to switch to dir and updates the script directory. +func (sh *Shell) ShowCmd(dir string, format string, args ...any) { + // Use the output lock directly so we can manage scriptDir. + sh.printLock.Lock() + defer sh.printLock.Unlock() + + cmd := sh.fmtCmd(dir, format, args...) + + if dir != "" && dir != "/" { + if dir != sh.scriptDir { + // Show changing to dir and update the current directory. + sh.printLocked(sh.fmtCmd("", "cd %s\n", dir)) + sh.scriptDir = dir + } + // Replace scriptDir is our working directory. Replace it + // with "." in the command. + dot := " ." + if dir[len(dir)-1] == filepath.Separator { + dot += string(filepath.Separator) + } + cmd = strings.ReplaceAll(" "+cmd, " "+dir, dot)[1:] + } + + sh.printLocked(cmd + "\n") +} + +// reportCmd reports the output and exit status of a command. The cmdOut and +// cmdErr arguments are the output and exit error of the command, respectively. +// +// The exact reporting behavior is as follows: +// +// cmdOut cmdErr Result +// "" nil print nothing, return nil +// !="" nil print output, return nil +// "" !=nil print nothing, return cmdErr (later printed) +// !="" !=nil print nothing, ignore err, return output as error (later printed) +// +// reportCmd returns a non-nil error if and only if cmdErr != nil. It assumes +// that the command output, if non-empty, is more detailed than the command +// error (which is usually just an exit status), so prefers using the output as +// the ultimate error. Typically, the caller should return this error from an +// Action, which it will be printed by the Builder. +// +// reportCmd formats the output as "# desc" followed by the given output. The +// output is expected to contain references to 'dir', usually the source +// directory for the package that has failed to build. reportCmd rewrites +// mentions of dir with a relative path to dir when the relative path is +// shorter. This is usually more pleasant. For example, if fmt doesn't compile +// and we are in src/html, the output is +// +// $ go build +// # fmt +// ../fmt/print.go:1090: undefined: asdf +// $ +// +// instead of +// +// $ go build +// # fmt +// /usr/gopher/go/src/fmt/print.go:1090: undefined: asdf +// $ +// +// reportCmd also replaces references to the work directory with $WORK, replaces +// cgo file paths with the original file path, and replaces cgo-mangled names +// with "C.name". +// +// desc is optional. If "", a.Package.Desc() is used. +// +// dir is optional. If "", a.Package.Dir is used. +func (sh *Shell) reportCmd(desc, dir string, cmdOut []byte, cmdErr error) error { + if len(cmdOut) == 0 && cmdErr == nil { + // Common case + return nil + } + if len(cmdOut) == 0 && cmdErr != nil { + // Just return the error. + // + // TODO: This is what we've done for a long time, but it may be a + // mistake because it loses all of the extra context and results in + // ultimately less descriptive output. We should probably just take the + // text of cmdErr as the output in this case and do everything we + // otherwise would. We could chain the errors if we feel like it. + return cmdErr + } + + // Fetch defaults from the package. + var p *load.Package + a := sh.action + if a != nil { + p = a.Package + } + var importPath string + if p != nil { + importPath = p.ImportPath + if desc == "" { + desc = p.Desc() + } + if dir == "" { + dir = p.Dir + } + } + + out := string(cmdOut) + + if !strings.HasSuffix(out, "\n") { + out = out + "\n" + } + + // Replace workDir with $WORK + out = replacePrefix(out, sh.workDir, "$WORK") + + // Rewrite mentions of dir with a relative path to dir + // when the relative path is shorter. + for { + // Note that dir starts out long, something like + // /foo/bar/baz/root/a + // The target string to be reduced is something like + // (blah-blah-blah) /foo/bar/baz/root/sibling/whatever.go:blah:blah + // /foo/bar/baz/root/a doesn't match /foo/bar/baz/root/sibling, but the prefix + // /foo/bar/baz/root does. And there may be other niblings sharing shorter + // prefixes, the only way to find them is to look. + // This doesn't always produce a relative path -- + // /foo is shorter than ../../.., for example. + if reldir := base.ShortPath(dir); reldir != dir { + out = replacePrefix(out, dir, reldir) + if filepath.Separator == '\\' { + // Don't know why, sometimes this comes out with slashes, not backslashes. + wdir := strings.ReplaceAll(dir, "\\", "/") + out = replacePrefix(out, wdir, reldir) + } + } + dirP := filepath.Dir(dir) + if dir == dirP { + break + } + dir = dirP + } + + // Fix up output referring to cgo-generated code to be more readable. + // Replace x.go:19[/tmp/.../x.cgo1.go:18] with x.go:19. + // Replace *[100]_Ctype_foo with *[100]C.foo. + // If we're using -x, assume we're debugging and want the full dump, so disable the rewrite. + if !cfg.BuildX && cgoLine.MatchString(out) { + out = cgoLine.ReplaceAllString(out, "") + out = cgoTypeSigRe.ReplaceAllString(out, "C.") + } + + // Usually desc is already p.Desc(), but if not, signal cmdError.Error to + // add a line explicitly metioning the import path. + needsPath := importPath != "" && p != nil && desc != p.Desc() + + err := &cmdError{desc, out, importPath, needsPath} + if cmdErr != nil { + // The command failed. Report the output up as an error. + return err + } + // The command didn't fail, so just print the output as appropriate. + if a != nil && a.output != nil { + // The Action is capturing output. + a.output = append(a.output, err.Error()...) + } else { + // Write directly to the Builder output. + sh.Print(err.Error()) + } + return nil +} + +// replacePrefix is like strings.ReplaceAll, but only replaces instances of old +// that are preceded by ' ', '\t', or appear at the beginning of a line. +func replacePrefix(s, old, new string) string { + n := strings.Count(s, old) + if n == 0 { + return s + } + + s = strings.ReplaceAll(s, " "+old, " "+new) + s = strings.ReplaceAll(s, "\n"+old, "\n"+new) + s = strings.ReplaceAll(s, "\n\t"+old, "\n\t"+new) + if strings.HasPrefix(s, old) { + s = new + s[len(old):] + } + return s +} + +type cmdError struct { + desc string + text string + importPath string + needsPath bool // Set if desc does not already include the import path +} + +func (e *cmdError) Error() string { + var msg string + if e.needsPath { + // Ensure the import path is part of the message. + // Clearly distinguish the description from the import path. + msg = fmt.Sprintf("# %s\n# [%s]\n", e.importPath, e.desc) + } else { + msg = "# " + e.desc + "\n" + } + return msg + e.text +} + +func (e *cmdError) ImportPath() string { + return e.importPath +} + +var cgoLine = lazyregexp.New(`\[[^\[\]]+\.(cgo1|cover)\.go:[0-9]+(:[0-9]+)?\]`) +var cgoTypeSigRe = lazyregexp.New(`\b_C2?(type|func|var|macro)_\B`) + +// run runs the command given by cmdline in the directory dir. +// If the command fails, run prints information about the failure +// and returns a non-nil error. +func (sh *Shell) run(dir string, desc string, env []string, cmdargs ...any) error { + out, err := sh.runOut(dir, env, cmdargs...) + if desc == "" { + desc = sh.fmtCmd(dir, "%s", strings.Join(str.StringList(cmdargs...), " ")) + } + return sh.reportCmd(desc, dir, out, err) +} + +// runOut runs the command given by cmdline in the directory dir. +// It returns the command output and any errors that occurred. +// It accumulates execution time in a. +func (sh *Shell) runOut(dir string, env []string, cmdargs ...any) ([]byte, error) { + a := sh.action + + cmdline := str.StringList(cmdargs...) + + for _, arg := range cmdline { + // GNU binutils commands, including gcc and gccgo, interpret an argument + // @foo anywhere in the command line (even following --) as meaning + // "read and insert arguments from the file named foo." + // Don't say anything that might be misinterpreted that way. + if strings.HasPrefix(arg, "@") { + return nil, fmt.Errorf("invalid command-line argument %s in command: %s", arg, joinUnambiguously(cmdline)) + } + } + + if cfg.BuildN || cfg.BuildX { + var envcmdline string + for _, e := range env { + if j := strings.IndexByte(e, '='); j != -1 { + if strings.ContainsRune(e[j+1:], '\'') { + envcmdline += fmt.Sprintf("%s=%q", e[:j], e[j+1:]) + } else { + envcmdline += fmt.Sprintf("%s='%s'", e[:j], e[j+1:]) + } + envcmdline += " " + } + } + envcmdline += joinUnambiguously(cmdline) + sh.ShowCmd(dir, "%s", envcmdline) + if cfg.BuildN { + return nil, nil + } + } + + var buf bytes.Buffer + path, err := cfg.LookPath(cmdline[0]) + if err != nil { + return nil, err + } + cmd := exec.Command(path, cmdline[1:]...) + if cmd.Path != "" { + cmd.Args[0] = cmd.Path + } + cmd.Stdout = &buf + cmd.Stderr = &buf + cleanup := passLongArgsInResponseFiles(cmd) + defer cleanup() + if dir != "." { + cmd.Dir = dir + } + cmd.Env = cmd.Environ() // Pre-allocate with correct PWD. + + // Add the TOOLEXEC_IMPORTPATH environment variable for -toolexec tools. + // It doesn't really matter if -toolexec isn't being used. + // Note that a.Package.Desc is not really an import path, + // but this is consistent with 'go list -f {{.ImportPath}}'. + // Plus, it is useful to uniquely identify packages in 'go list -json'. + if a != nil && a.Package != nil { + cmd.Env = append(cmd.Env, "TOOLEXEC_IMPORTPATH="+a.Package.Desc()) + } + + cmd.Env = append(cmd.Env, env...) + start := time.Now() + err = cmd.Run() + if a != nil && a.json != nil { + aj := a.json + aj.Cmd = append(aj.Cmd, joinUnambiguously(cmdline)) + aj.CmdReal += time.Since(start) + if ps := cmd.ProcessState; ps != nil { + aj.CmdUser += ps.UserTime() + aj.CmdSys += ps.SystemTime() + } + } + + // err can be something like 'exit status 1'. + // Add information about what program was running. + // Note that if buf.Bytes() is non-empty, the caller usually + // shows buf.Bytes() and does not print err at all, so the + // prefix here does not make most output any more verbose. + if err != nil { + err = errors.New(cmdline[0] + ": " + err.Error()) + } + return buf.Bytes(), err +} + +// joinUnambiguously prints the slice, quoting where necessary to make the +// output unambiguous. +// TODO: See issue 5279. The printing of commands needs a complete redo. +func joinUnambiguously(a []string) string { + var buf strings.Builder + for i, s := range a { + if i > 0 { + buf.WriteByte(' ') + } + q := strconv.Quote(s) + // A gccgo command line can contain -( and -). + // Make sure we quote them since they are special to the shell. + // The trimpath argument can also contain > (part of =>) and ;. Quote those too. + if s == "" || strings.ContainsAny(s, " ()>;") || len(q) > len(s)+2 { + buf.WriteString(q) + } else { + buf.WriteString(s) + } + } + return buf.String() +} diff --git a/src/cmd/go/internal/work/shell_test.go b/src/cmd/go/internal/work/shell_test.go new file mode 100644 index 0000000..24bef4e --- /dev/null +++ b/src/cmd/go/internal/work/shell_test.go @@ -0,0 +1,139 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix + +package work + +import ( + "bytes" + "internal/testenv" + "strings" + "testing" + "unicode" +) + +func FuzzSplitPkgConfigOutput(f *testing.F) { + testenv.MustHaveExecPath(f, "/bin/sh") + + f.Add([]byte(`$FOO`)) + f.Add([]byte(`\$FOO`)) + f.Add([]byte(`${FOO}`)) + f.Add([]byte(`\${FOO}`)) + f.Add([]byte(`$(/bin/false)`)) + f.Add([]byte(`\$(/bin/false)`)) + f.Add([]byte(`$((0))`)) + f.Add([]byte(`\$((0))`)) + f.Add([]byte(`unescaped space`)) + f.Add([]byte(`escaped\ space`)) + f.Add([]byte(`"unterminated quote`)) + f.Add([]byte(`'unterminated quote`)) + f.Add([]byte(`unterminated escape\`)) + f.Add([]byte(`"quote with unterminated escape\`)) + f.Add([]byte(`'quoted "double quotes"'`)) + f.Add([]byte(`"quoted 'single quotes'"`)) + f.Add([]byte(`"\$0"`)) + f.Add([]byte(`"\$\0"`)) + f.Add([]byte(`"\$"`)) + f.Add([]byte(`"\$ "`)) + + // Example positive inputs from TestSplitPkgConfigOutput. + // Some bare newlines have been removed so that the inputs + // are valid in the shell script we use for comparison. + f.Add([]byte(`-r:foo -L/usr/white\ space/lib -lfoo\ bar -lbar\ baz`)) + f.Add([]byte(`-lextra\ fun\ arg\\`)) + f.Add([]byte("\textra whitespace\r")) + f.Add([]byte(" \r ")) + f.Add([]byte(`"-r:foo" "-L/usr/white space/lib" "-lfoo bar" "-lbar baz"`)) + f.Add([]byte(`"-lextra fun arg\\"`)) + f.Add([]byte(`" \r\n\ "`)) + f.Add([]byte(`""`)) + f.Add([]byte(``)) + f.Add([]byte(`"\\"`)) + f.Add([]byte(`"\x"`)) + f.Add([]byte(`"\\x"`)) + f.Add([]byte(`'\\'`)) + f.Add([]byte(`'\x'`)) + f.Add([]byte(`"\\x"`)) + f.Add([]byte("\\\n")) + f.Add([]byte(`-fPIC -I/test/include/foo -DQUOTED='"/test/share/doc"'`)) + f.Add([]byte(`-fPIC -I/test/include/foo -DQUOTED="/test/share/doc"`)) + f.Add([]byte(`-fPIC -I/test/include/foo -DQUOTED=\"/test/share/doc\"`)) + f.Add([]byte(`-fPIC -I/test/include/foo -DQUOTED='/test/share/doc'`)) + f.Add([]byte(`-DQUOTED='/te\st/share/d\oc'`)) + f.Add([]byte(`-Dhello=10 -Dworld=+32 -DDEFINED_FROM_PKG_CONFIG=hello\ world`)) + f.Add([]byte(`"broken\"" \\\a "a"`)) + + // Example negative inputs from TestSplitPkgConfigOutput. + f.Add([]byte(`" \r\n `)) + f.Add([]byte(`"-r:foo" "-L/usr/white space/lib "-lfoo bar" "-lbar baz"`)) + f.Add([]byte(`"-lextra fun arg\\`)) + f.Add([]byte(`broken flag\`)) + f.Add([]byte(`extra broken flag \`)) + f.Add([]byte(`\`)) + f.Add([]byte(`"broken\"" "extra" \`)) + + f.Fuzz(func(t *testing.T, b []byte) { + t.Parallel() + + if bytes.ContainsAny(b, "*?[#~%\x00{}!") { + t.Skipf("skipping %#q: contains a sometimes-quoted character", b) + } + // splitPkgConfigOutput itself rejects inputs that contain unquoted + // shell operator characters. (Quoted shell characters are fine.) + + for _, c := range b { + if c > unicode.MaxASCII { + t.Skipf("skipping %#q: contains a non-ASCII character %q", b, c) + } + if !unicode.IsGraphic(rune(c)) && !unicode.IsSpace(rune(c)) { + t.Skipf("skipping %#q: contains non-graphic character %q", b, c) + } + } + + args, err := splitPkgConfigOutput(b) + if err != nil { + // We haven't checked that the shell would actually reject this input too, + // but if splitPkgConfigOutput rejected it it's probably too dangerous to + // run in the script. + t.Logf("%#q: %v", b, err) + return + } + t.Logf("splitPkgConfigOutput(%#q) = %#q", b, args) + if len(args) == 0 { + t.Skipf("skipping %#q: contains no arguments", b) + } + + var buf strings.Builder + for _, arg := range args { + buf.WriteString(arg) + buf.WriteString("\n") + } + wantOut := buf.String() + + if strings.Count(wantOut, "\n") != len(args)+bytes.Count(b, []byte("\n")) { + // One of the newlines in b was treated as a delimiter and not part of an + // argument. Our bash test script would interpret that as a syntax error. + t.Skipf("skipping %#q: contains a bare newline", b) + } + + // We use the printf shell command to echo the arguments because, per + // https://pubs.opengroup.org/onlinepubs/9699919799/utilities/echo.html#tag_20_37_16: + // “It is not possible to use echo portably across all POSIX systems unless + // both -n (as the first argument) and escape sequences are omitted.” + cmd := testenv.Command(t, "/bin/sh", "-c", "printf '%s\n' "+string(b)) + cmd.Env = append(cmd.Environ(), "LC_ALL=POSIX", "POSIXLY_CORRECT=1") + cmd.Stderr = new(strings.Builder) + out, err := cmd.Output() + if err != nil { + t.Fatalf("%#q: %v\n%s", cmd.Args, err, cmd.Stderr) + } + + if string(out) != wantOut { + t.Logf("%#q:\n%#q", cmd.Args, out) + t.Logf("want:\n%#q", wantOut) + t.Errorf("parsed args do not match") + } + }) +} |