diff options
Diffstat (limited to 'src/cmd/dist')
| -rw-r--r-- | src/cmd/dist/README | 27 | ||||
| -rw-r--r-- | src/cmd/dist/build.go | 1789 | ||||
| -rw-r--r-- | src/cmd/dist/buildgo.go | 127 | ||||
| -rw-r--r-- | src/cmd/dist/buildruntime.go | 105 | ||||
| -rw-r--r-- | src/cmd/dist/buildtag.go | 133 | ||||
| -rw-r--r-- | src/cmd/dist/buildtag_test.go | 43 | ||||
| -rw-r--r-- | src/cmd/dist/buildtool.go | 325 | ||||
| -rw-r--r-- | src/cmd/dist/doc.go | 19 | ||||
| -rw-r--r-- | src/cmd/dist/exec.go | 53 | ||||
| -rw-r--r-- | src/cmd/dist/imports.go | 276 | ||||
| -rw-r--r-- | src/cmd/dist/main.go | 189 | ||||
| -rw-r--r-- | src/cmd/dist/sys_default.go | 11 | ||||
| -rw-r--r-- | src/cmd/dist/sys_windows.go | 57 | ||||
| -rw-r--r-- | src/cmd/dist/test.go | 1684 | ||||
| -rw-r--r-- | src/cmd/dist/test_linux.go | 28 | ||||
| -rw-r--r-- | src/cmd/dist/util.go | 488 | ||||
| -rw-r--r-- | src/cmd/dist/util_gc.go | 21 | ||||
| -rw-r--r-- | src/cmd/dist/util_gccgo.go | 14 | ||||
| -rw-r--r-- | src/cmd/dist/vfp_arm.s | 27 | ||||
| -rw-r--r-- | src/cmd/dist/vfp_default.s | 17 |
20 files changed, 5433 insertions, 0 deletions
diff --git a/src/cmd/dist/README b/src/cmd/dist/README new file mode 100644 index 0000000..0649e88 --- /dev/null +++ b/src/cmd/dist/README @@ -0,0 +1,27 @@ +This program, dist, is the bootstrapping tool for the Go distribution. + +As of Go 1.5, dist and other parts of the compiler toolchain are written +in Go, making bootstrapping a little more involved than in the past. +The approach is to build the current release of Go with an earlier one. + +The process to install Go 1.x, for x ≥ 5, is: + +1. Build cmd/dist with Go 1.4. +2. Using dist, build Go 1.x compiler toolchain with Go 1.4. +3. Using dist, rebuild Go 1.x compiler toolchain with itself. +4. Using dist, build Go 1.x cmd/go (as go_bootstrap) with Go 1.x compiler toolchain. +5. Using go_bootstrap, build the remaining Go 1.x standard library and commands. + +NOTE: During the transition from the old C-based toolchain to the Go-based one, +step 2 also builds the parts of the toolchain written in C, and step 3 does not +recompile those. + +Because of backward compatibility, although the steps above say Go 1.4, +in practice any release ≥ Go 1.4 but < Go 1.x will work as the bootstrap base. + +See golang.org/s/go15bootstrap for more details. + +Compared to Go 1.4 and earlier, dist will also take over much of what used to +be done by make.bash/make.bat/make.rc and all of what used to be done by +run.bash/run.bat/run.rc, because it is nicer to implement that logic in Go +than in three different scripting languages simultaneously. diff --git a/src/cmd/dist/build.go b/src/cmd/dist/build.go new file mode 100644 index 0000000..d37c3f8 --- /dev/null +++ b/src/cmd/dist/build.go @@ -0,0 +1,1789 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + "time" +) + +// Initialization for any invocation. + +// The usual variables. +var ( + goarch string + gobin string + gohostarch string + gohostos string + goos string + goarm string + go386 string + goamd64 string + gomips string + gomips64 string + goppc64 string + goroot string + goroot_final string + goextlinkenabled string + gogcflags string // For running built compiler + goldflags string + goexperiment string + workdir string + tooldir string + oldgoos string + oldgoarch string + exe string + defaultcc map[string]string + defaultcxx map[string]string + defaultpkgconfig string + defaultldso string + + rebuildall bool + defaultclang bool + + vflag int // verbosity +) + +// The known architectures. +var okgoarch = []string{ + "386", + "amd64", + "arm", + "arm64", + "mips", + "mipsle", + "mips64", + "mips64le", + "ppc64", + "ppc64le", + "riscv64", + "s390x", + "sparc64", + "wasm", +} + +// The known operating systems. +var okgoos = []string{ + "darwin", + "dragonfly", + "illumos", + "ios", + "js", + "linux", + "android", + "solaris", + "freebsd", + "nacl", // keep; + "netbsd", + "openbsd", + "plan9", + "windows", + "aix", +} + +// find reports the first index of p in l[0:n], or else -1. +func find(p string, l []string) int { + for i, s := range l { + if p == s { + return i + } + } + return -1 +} + +// xinit handles initialization of the various global state, like goroot and goarch. +func xinit() { + b := os.Getenv("GOROOT") + if b == "" { + fatalf("$GOROOT must be set") + } + goroot = filepath.Clean(b) + + b = os.Getenv("GOROOT_FINAL") + if b == "" { + b = goroot + } + goroot_final = b + + b = os.Getenv("GOBIN") + if b == "" { + b = pathf("%s/bin", goroot) + } + gobin = b + + b = os.Getenv("GOOS") + if b == "" { + b = gohostos + } + goos = b + if find(goos, okgoos) < 0 { + fatalf("unknown $GOOS %s", goos) + } + + b = os.Getenv("GOARM") + if b == "" { + b = xgetgoarm() + } + goarm = b + + b = os.Getenv("GO386") + if b == "" { + b = "sse2" + } + go386 = b + + b = os.Getenv("GOAMD64") + if b == "" { + b = "v1" + } + goamd64 = b + + b = os.Getenv("GOMIPS") + if b == "" { + b = "hardfloat" + } + gomips = b + + b = os.Getenv("GOMIPS64") + if b == "" { + b = "hardfloat" + } + gomips64 = b + + b = os.Getenv("GOPPC64") + if b == "" { + b = "power8" + } + goppc64 = b + + if p := pathf("%s/src/all.bash", goroot); !isfile(p) { + fatalf("$GOROOT is not set correctly or not exported\n"+ + "\tGOROOT=%s\n"+ + "\t%s does not exist", goroot, p) + } + + b = os.Getenv("GOHOSTARCH") + if b != "" { + gohostarch = b + } + if find(gohostarch, okgoarch) < 0 { + fatalf("unknown $GOHOSTARCH %s", gohostarch) + } + + b = os.Getenv("GOARCH") + if b == "" { + b = gohostarch + } + goarch = b + if find(goarch, okgoarch) < 0 { + fatalf("unknown $GOARCH %s", goarch) + } + + b = os.Getenv("GO_EXTLINK_ENABLED") + if b != "" { + if b != "0" && b != "1" { + fatalf("unknown $GO_EXTLINK_ENABLED %s", b) + } + goextlinkenabled = b + } + + goexperiment = os.Getenv("GOEXPERIMENT") + // TODO(mdempsky): Validate known experiments? + + gogcflags = os.Getenv("BOOT_GO_GCFLAGS") + goldflags = os.Getenv("BOOT_GO_LDFLAGS") + + cc, cxx := "gcc", "g++" + if defaultclang { + cc, cxx = "clang", "clang++" + } + defaultcc = compilerEnv("CC", cc) + defaultcxx = compilerEnv("CXX", cxx) + + b = os.Getenv("PKG_CONFIG") + if b == "" { + b = "pkg-config" + } + defaultpkgconfig = b + + defaultldso = os.Getenv("GO_LDSO") + + // For tools being invoked but also for os.ExpandEnv. + os.Setenv("GO386", go386) + os.Setenv("GOAMD64", goamd64) + os.Setenv("GOARCH", goarch) + os.Setenv("GOARM", goarm) + os.Setenv("GOHOSTARCH", gohostarch) + os.Setenv("GOHOSTOS", gohostos) + os.Setenv("GOOS", goos) + os.Setenv("GOMIPS", gomips) + os.Setenv("GOMIPS64", gomips64) + os.Setenv("GOPPC64", goppc64) + os.Setenv("GOROOT", goroot) + os.Setenv("GOROOT_FINAL", goroot_final) + + // Use a build cache separate from the default user one. + // Also one that will be wiped out during startup, so that + // make.bash really does start from a clean slate. + os.Setenv("GOCACHE", pathf("%s/pkg/obj/go-build", goroot)) + + // Make the environment more predictable. + os.Setenv("LANG", "C") + os.Setenv("LANGUAGE", "en_US.UTF8") + + workdir = xworkdir() + if err := ioutil.WriteFile(pathf("%s/go.mod", workdir), []byte("module bootstrap"), 0666); err != nil { + fatalf("cannot write stub go.mod: %s", err) + } + xatexit(rmworkdir) + + tooldir = pathf("%s/pkg/tool/%s_%s", goroot, gohostos, gohostarch) +} + +// compilerEnv returns a map from "goos/goarch" to the +// compiler setting to use for that platform. +// The entry for key "" covers any goos/goarch not explicitly set in the map. +// For example, compilerEnv("CC", "gcc") returns the C compiler settings +// read from $CC, defaulting to gcc. +// +// The result is a map because additional environment variables +// can be set to change the compiler based on goos/goarch settings. +// The following applies to all envNames but CC is assumed to simplify +// the presentation. +// +// If no environment variables are set, we use def for all goos/goarch. +// $CC, if set, applies to all goos/goarch but is overridden by the following. +// $CC_FOR_TARGET, if set, applies to all goos/goarch except gohostos/gohostarch, +// but is overridden by the following. +// If gohostos=goos and gohostarch=goarch, then $CC_FOR_TARGET applies even for gohostos/gohostarch. +// $CC_FOR_goos_goarch, if set, applies only to goos/goarch. +func compilerEnv(envName, def string) map[string]string { + m := map[string]string{"": def} + + if env := os.Getenv(envName); env != "" { + m[""] = env + } + if env := os.Getenv(envName + "_FOR_TARGET"); env != "" { + if gohostos != goos || gohostarch != goarch { + m[gohostos+"/"+gohostarch] = m[""] + } + m[""] = env + } + + for _, goos := range okgoos { + for _, goarch := range okgoarch { + if env := os.Getenv(envName + "_FOR_" + goos + "_" + goarch); env != "" { + m[goos+"/"+goarch] = env + } + } + } + + return m +} + +// compilerEnvLookup returns the compiler settings for goos/goarch in map m. +func compilerEnvLookup(m map[string]string, goos, goarch string) string { + if cc := m[goos+"/"+goarch]; cc != "" { + return cc + } + return m[""] +} + +// rmworkdir deletes the work directory. +func rmworkdir() { + if vflag > 1 { + errprintf("rm -rf %s\n", workdir) + } + xremoveall(workdir) +} + +// Remove trailing spaces. +func chomp(s string) string { + return strings.TrimRight(s, " \t\r\n") +} + +func branchtag(branch string) (tag string, precise bool) { + log := run(goroot, CheckExit, "git", "log", "--decorate=full", "--format=format:%d", "master.."+branch) + tag = branch + for row, line := range strings.Split(log, "\n") { + // Each line is either blank, or looks like + // (tag: refs/tags/go1.4rc2, refs/remotes/origin/release-branch.go1.4, refs/heads/release-branch.go1.4) + // We need to find an element starting with refs/tags/. + const s = " refs/tags/" + i := strings.Index(line, s) + if i < 0 { + continue + } + // Trim off known prefix. + line = line[i+len(s):] + // The tag name ends at a comma or paren. + j := strings.IndexAny(line, ",)") + if j < 0 { + continue // malformed line; ignore it + } + tag = line[:j] + if row == 0 { + precise = true // tag denotes HEAD + } + break + } + return +} + +// findgoversion determines the Go version to use in the version string. +func findgoversion() string { + // The $GOROOT/VERSION file takes priority, for distributions + // without the source repo. + path := pathf("%s/VERSION", goroot) + if isfile(path) { + b := chomp(readfile(path)) + // Commands such as "dist version > VERSION" will cause + // the shell to create an empty VERSION file and set dist's + // stdout to its fd. dist in turn looks at VERSION and uses + // its content if available, which is empty at this point. + // Only use the VERSION file if it is non-empty. + if b != "" { + // Some builders cross-compile the toolchain on linux-amd64 + // and then copy the toolchain to the target builder (say, linux-arm) + // for use there. But on non-release (devel) branches, the compiler + // used on linux-amd64 will be an amd64 binary, and the compiler + // shipped to linux-arm will be an arm binary, so they will have different + // content IDs (they are binaries for different architectures) and so the + // packages compiled by the running-on-amd64 compiler will appear + // stale relative to the running-on-arm compiler. Avoid this by setting + // the version string to something that doesn't begin with devel. + // Then the version string will be used in place of the content ID, + // and the packages will look up-to-date. + // TODO(rsc): Really the builders could be writing out a better VERSION file instead, + // but it is easier to change cmd/dist than to try to make changes to + // the builder while Brad is away. + if strings.HasPrefix(b, "devel") { + if hostType := os.Getenv("META_BUILDLET_HOST_TYPE"); strings.Contains(hostType, "-cross") { + fmt.Fprintf(os.Stderr, "warning: changing VERSION from %q to %q\n", b, "builder "+hostType) + b = "builder " + hostType + } + } + return b + } + } + + // The $GOROOT/VERSION.cache file is a cache to avoid invoking + // git every time we run this command. Unlike VERSION, it gets + // deleted by the clean command. + path = pathf("%s/VERSION.cache", goroot) + if isfile(path) { + return chomp(readfile(path)) + } + + // Show a nicer error message if this isn't a Git repo. + if !isGitRepo() { + fatalf("FAILED: not a Git repo; must put a VERSION file in $GOROOT") + } + + // Otherwise, use Git. + // What is the current branch? + branch := chomp(run(goroot, CheckExit, "git", "rev-parse", "--abbrev-ref", "HEAD")) + + // What are the tags along the current branch? + tag := "devel" + precise := false + + // If we're on a release branch, use the closest matching tag + // that is on the release branch (and not on the master branch). + if strings.HasPrefix(branch, "release-branch.") { + tag, precise = branchtag(branch) + } + + if !precise { + // Tag does not point at HEAD; add 1.x base version, hash, and date to version. + // + // Note that we lightly parse internal/goversion/goversion.go to + // obtain the base version. We can't just import the package, + // because cmd/dist is built with a bootstrap GOROOT which could + // be an entirely different version of Go, like 1.4. We assume + // that the file contains "const Version = <Integer>". + + goversionSource := readfile(pathf("%s/src/internal/goversion/goversion.go", goroot)) + m := regexp.MustCompile(`(?m)^const Version = (\d+)`).FindStringSubmatch(goversionSource) + if m == nil { + fatalf("internal/goversion/goversion.go does not contain 'const Version = ...'") + } + tag += fmt.Sprintf(" go1.%s-", m[1]) + + tag += chomp(run(goroot, CheckExit, "git", "log", "-n", "1", "--format=format:%h %cd", "HEAD")) + } + + // Cache version. + writefile(tag, path, 0) + + return tag +} + +// isGitRepo reports whether the working directory is inside a Git repository. +func isGitRepo() bool { + // NB: simply checking the exit code of `git rev-parse --git-dir` would + // suffice here, but that requires deviating from the infrastructure + // provided by `run`. + gitDir := chomp(run(goroot, 0, "git", "rev-parse", "--git-dir")) + if !filepath.IsAbs(gitDir) { + gitDir = filepath.Join(goroot, gitDir) + } + return isdir(gitDir) +} + +/* + * Initial tree setup. + */ + +// The old tools that no longer live in $GOBIN or $GOROOT/bin. +var oldtool = []string{ + "5a", "5c", "5g", "5l", + "6a", "6c", "6g", "6l", + "8a", "8c", "8g", "8l", + "9a", "9c", "9g", "9l", + "6cov", + "6nm", + "6prof", + "cgo", + "ebnflint", + "goapi", + "gofix", + "goinstall", + "gomake", + "gopack", + "gopprof", + "gotest", + "gotype", + "govet", + "goyacc", + "quietgcc", +} + +// Unreleased directories (relative to $GOROOT) that should +// not be in release branches. +var unreleased = []string{ + "src/cmd/newlink", + "src/cmd/objwriter", + "src/debug/goobj", + "src/old", +} + +// setup sets up the tree for the initial build. +func setup() { + // Create bin directory. + if p := pathf("%s/bin", goroot); !isdir(p) { + xmkdir(p) + } + + // Create package directory. + if p := pathf("%s/pkg", goroot); !isdir(p) { + xmkdir(p) + } + + p := pathf("%s/pkg/%s_%s", goroot, gohostos, gohostarch) + if rebuildall { + xremoveall(p) + } + xmkdirall(p) + + if goos != gohostos || goarch != gohostarch { + p := pathf("%s/pkg/%s_%s", goroot, goos, goarch) + if rebuildall { + xremoveall(p) + } + xmkdirall(p) + } + + // Create object directory. + // We used to use it for C objects. + // Now we use it for the build cache, to separate dist's cache + // from any other cache the user might have. + p = pathf("%s/pkg/obj/go-build", goroot) + if rebuildall { + xremoveall(p) + } + xmkdirall(p) + xatexit(func() { xremoveall(p) }) + + // Create tool directory. + // We keep it in pkg/, just like the object directory above. + if rebuildall { + xremoveall(tooldir) + } + xmkdirall(tooldir) + + // Remove tool binaries from before the tool/gohostos_gohostarch + xremoveall(pathf("%s/bin/tool", goroot)) + + // Remove old pre-tool binaries. + for _, old := range oldtool { + xremove(pathf("%s/bin/%s", goroot, old)) + } + + // If $GOBIN is set and has a Go compiler, it must be cleaned. + for _, char := range "56789" { + if isfile(pathf("%s/%c%s", gobin, char, "g")) { + for _, old := range oldtool { + xremove(pathf("%s/%s", gobin, old)) + } + break + } + } + + // For release, make sure excluded things are excluded. + goversion := findgoversion() + if strings.HasPrefix(goversion, "release.") || (strings.HasPrefix(goversion, "go") && !strings.Contains(goversion, "beta")) { + for _, dir := range unreleased { + if p := pathf("%s/%s", goroot, dir); isdir(p) { + fatalf("%s should not exist in release build", p) + } + } + } +} + +/* + * Tool building + */ + +// deptab lists changes to the default dependencies for a given prefix. +// deps ending in /* read the whole directory; deps beginning with - +// exclude files with that prefix. +// Note that this table applies only to the build of cmd/go, +// after the main compiler bootstrap. +var deptab = []struct { + prefix string // prefix of target + dep []string // dependency tweaks for targets with that prefix +}{ + {"cmd/go/internal/cfg", []string{ + "zdefaultcc.go", + "zosarch.go", + }}, + {"runtime/internal/sys", []string{ + "zversion.go", + }}, + {"go/build", []string{ + "zcgo.go", + }}, +} + +// depsuffix records the allowed suffixes for source files. +var depsuffix = []string{ + ".s", + ".go", +} + +// gentab records how to generate some trivial files. +var gentab = []struct { + nameprefix string + gen func(string, string) +}{ + {"zdefaultcc.go", mkzdefaultcc}, + {"zosarch.go", mkzosarch}, + {"zversion.go", mkzversion}, + {"zcgo.go", mkzcgo}, + + // not generated anymore, but delete the file if we see it + {"enam.c", nil}, + {"anames5.c", nil}, + {"anames6.c", nil}, + {"anames8.c", nil}, + {"anames9.c", nil}, +} + +// installed maps from a dir name (as given to install) to a chan +// closed when the dir's package is installed. +var installed = make(map[string]chan struct{}) +var installedMu sync.Mutex + +func install(dir string) { + <-startInstall(dir) +} + +func startInstall(dir string) chan struct{} { + installedMu.Lock() + ch := installed[dir] + if ch == nil { + ch = make(chan struct{}) + installed[dir] = ch + go runInstall(dir, ch) + } + installedMu.Unlock() + return ch +} + +// runInstall installs the library, package, or binary associated with dir, +// which is relative to $GOROOT/src. +func runInstall(pkg string, ch chan struct{}) { + if pkg == "net" || pkg == "os/user" || pkg == "crypto/x509" { + fatalf("go_bootstrap cannot depend on cgo package %s", pkg) + } + + defer close(ch) + + if pkg == "unsafe" { + return + } + + if vflag > 0 { + if goos != gohostos || goarch != gohostarch { + errprintf("%s (%s/%s)\n", pkg, goos, goarch) + } else { + errprintf("%s\n", pkg) + } + } + + workdir := pathf("%s/%s", workdir, pkg) + xmkdirall(workdir) + + var clean []string + defer func() { + for _, name := range clean { + xremove(name) + } + }() + + // dir = full path to pkg. + dir := pathf("%s/src/%s", goroot, pkg) + name := filepath.Base(dir) + + // ispkg predicts whether the package should be linked as a binary, based + // on the name. There should be no "main" packages in vendor, since + // 'go mod vendor' will only copy imported packages there. + ispkg := !strings.HasPrefix(pkg, "cmd/") || strings.Contains(pkg, "/internal/") || strings.Contains(pkg, "/vendor/") + + // Start final link command line. + // Note: code below knows that link.p[targ] is the target. + var ( + link []string + targ int + ispackcmd bool + ) + if ispkg { + // Go library (package). + ispackcmd = true + link = []string{"pack", packagefile(pkg)} + targ = len(link) - 1 + xmkdirall(filepath.Dir(link[targ])) + } else { + // Go command. + elem := name + if elem == "go" { + elem = "go_bootstrap" + } + link = []string{pathf("%s/link", tooldir)} + if goos == "android" { + link = append(link, "-buildmode=pie") + } + if goldflags != "" { + link = append(link, goldflags) + } + link = append(link, "-extld="+compilerEnvLookup(defaultcc, goos, goarch)) + link = append(link, "-o", pathf("%s/%s%s", tooldir, elem, exe)) + targ = len(link) - 1 + } + ttarg := mtime(link[targ]) + + // Gather files that are sources for this target. + // Everything in that directory, and any target-specific + // additions. + files := xreaddir(dir) + + // Remove files beginning with . or _, + // which are likely to be editor temporary files. + // This is the same heuristic build.ScanDir uses. + // There do exist real C files beginning with _, + // so limit that check to just Go files. + files = filter(files, func(p string) bool { + return !strings.HasPrefix(p, ".") && (!strings.HasPrefix(p, "_") || !strings.HasSuffix(p, ".go")) + }) + + for _, dt := range deptab { + if pkg == dt.prefix || strings.HasSuffix(dt.prefix, "/") && strings.HasPrefix(pkg, dt.prefix) { + for _, p := range dt.dep { + p = os.ExpandEnv(p) + files = append(files, p) + } + } + } + files = uniq(files) + + // Convert to absolute paths. + for i, p := range files { + if !filepath.IsAbs(p) { + files[i] = pathf("%s/%s", dir, p) + } + } + + // Is the target up-to-date? + var gofiles, sfiles, missing []string + stale := rebuildall + files = filter(files, func(p string) bool { + for _, suf := range depsuffix { + if strings.HasSuffix(p, suf) { + goto ok + } + } + return false + ok: + t := mtime(p) + if !t.IsZero() && !strings.HasSuffix(p, ".a") && !shouldbuild(p, pkg) { + return false + } + if strings.HasSuffix(p, ".go") { + gofiles = append(gofiles, p) + } else if strings.HasSuffix(p, ".s") { + sfiles = append(sfiles, p) + } + if t.After(ttarg) { + stale = true + } + if t.IsZero() { + missing = append(missing, p) + } + return true + }) + + // If there are no files to compile, we're done. + if len(files) == 0 { + return + } + + if !stale { + return + } + + // For package runtime, copy some files into the work space. + if pkg == "runtime" { + xmkdirall(pathf("%s/pkg/include", goroot)) + // For use by assembly and C files. + copyfile(pathf("%s/pkg/include/textflag.h", goroot), + pathf("%s/src/runtime/textflag.h", goroot), 0) + copyfile(pathf("%s/pkg/include/funcdata.h", goroot), + pathf("%s/src/runtime/funcdata.h", goroot), 0) + copyfile(pathf("%s/pkg/include/asm_ppc64x.h", goroot), + pathf("%s/src/runtime/asm_ppc64x.h", goroot), 0) + } + + // Generate any missing files; regenerate existing ones. + for _, p := range files { + elem := filepath.Base(p) + for _, gt := range gentab { + if gt.gen == nil { + continue + } + if strings.HasPrefix(elem, gt.nameprefix) { + if vflag > 1 { + errprintf("generate %s\n", p) + } + gt.gen(dir, p) + // Do not add generated file to clean list. + // In runtime, we want to be able to + // build the package with the go tool, + // and it assumes these generated files already + // exist (it does not know how to build them). + // The 'clean' command can remove + // the generated files. + goto built + } + } + // Did not rebuild p. + if find(p, missing) >= 0 { + fatalf("missing file %s", p) + } + built: + } + + // Resolve imported packages to actual package paths. + // Make sure they're installed. + importMap := make(map[string]string) + for _, p := range gofiles { + for _, imp := range readimports(p) { + if imp == "C" { + fatalf("%s imports C", p) + } + importMap[imp] = resolveVendor(imp, dir) + } + } + sortedImports := make([]string, 0, len(importMap)) + for imp := range importMap { + sortedImports = append(sortedImports, imp) + } + sort.Strings(sortedImports) + + for _, dep := range importMap { + if dep == "C" { + fatalf("%s imports C", pkg) + } + startInstall(dep) + } + for _, dep := range importMap { + install(dep) + } + + if goos != gohostos || goarch != gohostarch { + // We've generated the right files; the go command can do the build. + if vflag > 1 { + errprintf("skip build for cross-compile %s\n", pkg) + } + return + } + + asmArgs := []string{ + pathf("%s/asm", tooldir), + "-I", workdir, + "-I", pathf("%s/pkg/include", goroot), + "-D", "GOOS_" + goos, + "-D", "GOARCH_" + goarch, + "-D", "GOOS_GOARCH_" + goos + "_" + goarch, + "-p", pkg, + } + if goarch == "mips" || goarch == "mipsle" { + // Define GOMIPS_value from gomips. + asmArgs = append(asmArgs, "-D", "GOMIPS_"+gomips) + } + if goarch == "mips64" || goarch == "mips64le" { + // Define GOMIPS64_value from gomips64. + asmArgs = append(asmArgs, "-D", "GOMIPS64_"+gomips64) + } + goasmh := pathf("%s/go_asm.h", workdir) + if IsRuntimePackagePath(pkg) { + asmArgs = append(asmArgs, "-compiling-runtime") + } + + // Collect symabis from assembly code. + var symabis string + if len(sfiles) > 0 { + symabis = pathf("%s/symabis", workdir) + var wg sync.WaitGroup + asmabis := append(asmArgs[:len(asmArgs):len(asmArgs)], "-gensymabis", "-o", symabis) + asmabis = append(asmabis, sfiles...) + if err := ioutil.WriteFile(goasmh, nil, 0666); err != nil { + fatalf("cannot write empty go_asm.h: %s", err) + } + bgrun(&wg, dir, asmabis...) + bgwait(&wg) + } + + // Build an importcfg file for the compiler. + buf := &bytes.Buffer{} + for _, imp := range sortedImports { + if imp == "unsafe" { + continue + } + dep := importMap[imp] + if imp != dep { + fmt.Fprintf(buf, "importmap %s=%s\n", imp, dep) + } + fmt.Fprintf(buf, "packagefile %s=%s\n", dep, packagefile(dep)) + } + importcfg := pathf("%s/importcfg", workdir) + if err := ioutil.WriteFile(importcfg, buf.Bytes(), 0666); err != nil { + fatalf("cannot write importcfg file: %v", err) + } + + var archive string + // The next loop will compile individual non-Go files. + // Hand the Go files to the compiler en masse. + // For packages containing assembly, this writes go_asm.h, which + // the assembly files will need. + pkgName := pkg + if strings.HasPrefix(pkg, "cmd/") && strings.Count(pkg, "/") == 1 { + pkgName = "main" + } + b := pathf("%s/_go_.a", workdir) + clean = append(clean, b) + if !ispackcmd { + link = append(link, b) + } else { + archive = b + } + + // Compile Go code. + compile := []string{pathf("%s/compile", tooldir), "-std", "-pack", "-o", b, "-p", pkgName, "-importcfg", importcfg} + if gogcflags != "" { + compile = append(compile, strings.Fields(gogcflags)...) + } + if pkg == "runtime" { + compile = append(compile, "-+") + } + if len(sfiles) > 0 { + compile = append(compile, "-asmhdr", goasmh) + } + if symabis != "" { + compile = append(compile, "-symabis", symabis) + } + if goos == "android" { + compile = append(compile, "-shared") + } + + compile = append(compile, gofiles...) + var wg sync.WaitGroup + // We use bgrun and immediately wait for it instead of calling run() synchronously. + // This executes all jobs through the bgwork channel and allows the process + // to exit cleanly in case an error occurs. + bgrun(&wg, dir, compile...) + bgwait(&wg) + + // Compile the files. + for _, p := range sfiles { + // Assembly file for a Go package. + compile := asmArgs[:len(asmArgs):len(asmArgs)] + + doclean := true + b := pathf("%s/%s", workdir, filepath.Base(p)) + + // Change the last character of the output file (which was c or s). + b = b[:len(b)-1] + "o" + compile = append(compile, "-o", b, p) + bgrun(&wg, dir, compile...) + + link = append(link, b) + if doclean { + clean = append(clean, b) + } + } + bgwait(&wg) + + if ispackcmd { + xremove(link[targ]) + dopack(link[targ], archive, link[targ+1:]) + return + } + + // Remove target before writing it. + xremove(link[targ]) + bgrun(&wg, "", link...) + bgwait(&wg) +} + +// packagefile returns the path to a compiled .a file for the given package +// path. Paths may need to be resolved with resolveVendor first. +func packagefile(pkg string) string { + return pathf("%s/pkg/%s_%s/%s.a", goroot, goos, goarch, pkg) +} + +// matchtag reports whether the tag matches this build. +func matchtag(tag string) bool { + return tag == "gc" || tag == goos || tag == goarch || tag == "cmd_go_bootstrap" || tag == "go1.1" || + (goos == "android" && tag == "linux") || + (goos == "illumos" && tag == "solaris") || + (goos == "ios" && tag == "darwin") +} + +// shouldbuild reports whether we should build this file. +// It applies the same rules that are used with context tags +// in package go/build, except it's less picky about the order +// of GOOS and GOARCH. +// We also allow the special tag cmd_go_bootstrap. +// See ../go/bootstrap.go and package go/build. +func shouldbuild(file, pkg string) bool { + // Check file name for GOOS or GOARCH. + name := filepath.Base(file) + excluded := func(list []string, ok string) bool { + for _, x := range list { + if x == ok || (ok == "android" && x == "linux") || (ok == "illumos" && x == "solaris") || (ok == "ios" && x == "darwin") { + continue + } + i := strings.Index(name, x) + if i <= 0 || name[i-1] != '_' { + continue + } + i += len(x) + if i == len(name) || name[i] == '.' || name[i] == '_' { + return true + } + } + return false + } + if excluded(okgoos, goos) || excluded(okgoarch, goarch) { + return false + } + + // Omit test files. + if strings.Contains(name, "_test") { + return false + } + + // Check file contents for //go:build lines. + for _, p := range strings.Split(readfile(file), "\n") { + p = strings.TrimSpace(p) + if p == "" { + continue + } + code := p + i := strings.Index(code, "//") + if i > 0 { + code = strings.TrimSpace(code[:i]) + } + if code == "package documentation" { + return false + } + if code == "package main" && pkg != "cmd/go" && pkg != "cmd/cgo" { + return false + } + if !strings.HasPrefix(p, "//") { + break + } + if strings.HasPrefix(p, "//go:build ") { + matched, err := matchexpr(p[len("//go:build "):]) + if err != nil { + errprintf("%s: %v", file, err) + } + return matched + } + } + + return true +} + +// copy copies the file src to dst, via memory (so only good for small files). +func copyfile(dst, src string, flag int) { + if vflag > 1 { + errprintf("cp %s %s\n", src, dst) + } + writefile(readfile(src), dst, flag) +} + +// dopack copies the package src to dst, +// appending the files listed in extra. +// The archive format is the traditional Unix ar format. +func dopack(dst, src string, extra []string) { + bdst := bytes.NewBufferString(readfile(src)) + for _, file := range extra { + b := readfile(file) + // find last path element for archive member name + i := strings.LastIndex(file, "/") + 1 + j := strings.LastIndex(file, `\`) + 1 + if i < j { + i = j + } + fmt.Fprintf(bdst, "%-16.16s%-12d%-6d%-6d%-8o%-10d`\n", file[i:], 0, 0, 0, 0644, len(b)) + bdst.WriteString(b) + if len(b)&1 != 0 { + bdst.WriteByte(0) + } + } + writefile(bdst.String(), dst, 0) +} + +var runtimegen = []string{ + "zaexperiment.h", + "zversion.go", +} + +// cleanlist is a list of packages with generated files and commands. +var cleanlist = []string{ + "runtime/internal/sys", + "cmd/cgo", + "cmd/go/internal/cfg", + "go/build", +} + +func clean() { + for _, name := range cleanlist { + path := pathf("%s/src/%s", goroot, name) + // Remove generated files. + for _, elem := range xreaddir(path) { + for _, gt := range gentab { + if strings.HasPrefix(elem, gt.nameprefix) { + xremove(pathf("%s/%s", path, elem)) + } + } + } + // Remove generated binary named for directory. + if strings.HasPrefix(name, "cmd/") { + xremove(pathf("%s/%s", path, name[4:])) + } + } + + // remove runtimegen files. + path := pathf("%s/src/runtime", goroot) + for _, elem := range runtimegen { + xremove(pathf("%s/%s", path, elem)) + } + + if rebuildall { + // Remove object tree. + xremoveall(pathf("%s/pkg/obj/%s_%s", goroot, gohostos, gohostarch)) + + // Remove installed packages and tools. + xremoveall(pathf("%s/pkg/%s_%s", goroot, gohostos, gohostarch)) + xremoveall(pathf("%s/pkg/%s_%s", goroot, goos, goarch)) + xremoveall(pathf("%s/pkg/%s_%s_race", goroot, gohostos, gohostarch)) + xremoveall(pathf("%s/pkg/%s_%s_race", goroot, goos, goarch)) + xremoveall(tooldir) + + // Remove cached version info. + xremove(pathf("%s/VERSION.cache", goroot)) + } +} + +/* + * command implementations + */ + +// The env command prints the default environment. +func cmdenv() { + path := flag.Bool("p", false, "emit updated PATH") + plan9 := flag.Bool("9", false, "emit plan 9 syntax") + windows := flag.Bool("w", false, "emit windows syntax") + xflagparse(0) + + format := "%s=\"%s\"\n" + switch { + case *plan9: + format = "%s='%s'\n" + case *windows: + format = "set %s=%s\r\n" + } + + xprintf(format, "GOARCH", goarch) + xprintf(format, "GOBIN", gobin) + xprintf(format, "GOCACHE", os.Getenv("GOCACHE")) + xprintf(format, "GODEBUG", os.Getenv("GODEBUG")) + xprintf(format, "GOHOSTARCH", gohostarch) + xprintf(format, "GOHOSTOS", gohostos) + xprintf(format, "GOOS", goos) + xprintf(format, "GOPROXY", os.Getenv("GOPROXY")) + xprintf(format, "GOROOT", goroot) + xprintf(format, "GOTMPDIR", os.Getenv("GOTMPDIR")) + xprintf(format, "GOTOOLDIR", tooldir) + if goarch == "arm" { + xprintf(format, "GOARM", goarm) + } + if goarch == "386" { + xprintf(format, "GO386", go386) + } + if goarch == "amd64" { + xprintf(format, "GOAMD64", goamd64) + } + if goarch == "mips" || goarch == "mipsle" { + xprintf(format, "GOMIPS", gomips) + } + if goarch == "mips64" || goarch == "mips64le" { + xprintf(format, "GOMIPS64", gomips64) + } + if goarch == "ppc64" || goarch == "ppc64le" { + xprintf(format, "GOPPC64", goppc64) + } + + if *path { + sep := ":" + if gohostos == "windows" { + sep = ";" + } + xprintf(format, "PATH", fmt.Sprintf("%s%s%s", gobin, sep, os.Getenv("PATH"))) + } +} + +var ( + timeLogEnabled = os.Getenv("GOBUILDTIMELOGFILE") != "" + timeLogMu sync.Mutex + timeLogFile *os.File + timeLogStart time.Time +) + +func timelog(op, name string) { + if !timeLogEnabled { + return + } + timeLogMu.Lock() + defer timeLogMu.Unlock() + if timeLogFile == nil { + f, err := os.OpenFile(os.Getenv("GOBUILDTIMELOGFILE"), os.O_RDWR|os.O_APPEND, 0666) + if err != nil { + log.Fatal(err) + } + buf := make([]byte, 100) + n, _ := f.Read(buf) + s := string(buf[:n]) + if i := strings.Index(s, "\n"); i >= 0 { + s = s[:i] + } + i := strings.Index(s, " start") + if i < 0 { + log.Fatalf("time log %s does not begin with start line", os.Getenv("GOBUILDTIMELOGFILE")) + } + t, err := time.Parse(time.UnixDate, s[:i]) + if err != nil { + log.Fatalf("cannot parse time log line %q: %v", s, err) + } + timeLogStart = t + timeLogFile = f + } + t := time.Now() + fmt.Fprintf(timeLogFile, "%s %+.1fs %s %s\n", t.Format(time.UnixDate), t.Sub(timeLogStart).Seconds(), op, name) +} + +var toolchain = []string{"cmd/asm", "cmd/cgo", "cmd/compile", "cmd/link"} + +// The bootstrap command runs a build from scratch, +// stopping at having installed the go_bootstrap command. +// +// WARNING: This command runs after cmd/dist is built with Go 1.4. +// It rebuilds and installs cmd/dist with the new toolchain, so other +// commands (like "go tool dist test" in run.bash) can rely on bug fixes +// made since Go 1.4, but this function cannot. In particular, the uses +// of os/exec in this function cannot assume that +// cmd.Env = append(os.Environ(), "X=Y") +// sets $X to Y in the command's environment. That guarantee was +// added after Go 1.4, and in fact in Go 1.4 it was typically the opposite: +// if $X was already present in os.Environ(), most systems preferred +// that setting, not the new one. +func cmdbootstrap() { + timelog("start", "dist bootstrap") + defer timelog("end", "dist bootstrap") + + var noBanner, noClean bool + var debug bool + flag.BoolVar(&rebuildall, "a", rebuildall, "rebuild all") + flag.BoolVar(&debug, "d", debug, "enable debugging of bootstrap process") + flag.BoolVar(&noBanner, "no-banner", noBanner, "do not print banner") + flag.BoolVar(&noClean, "no-clean", noClean, "print deprecation warning") + + xflagparse(0) + + if noClean { + xprintf("warning: --no-clean is deprecated and has no effect; use 'go install std cmd' instead\n") + } + + // Set GOPATH to an internal directory. We shouldn't actually + // need to store files here, since the toolchain won't + // depend on modules outside of vendor directories, but if + // GOPATH points somewhere else (e.g., to GOROOT), the + // go tool may complain. + os.Setenv("GOPATH", pathf("%s/pkg/obj/gopath", goroot)) + + // Disable GOEXPERIMENT when building toolchain1 and + // go_bootstrap. We don't need any experiments for the + // bootstrap toolchain, and this lets us avoid duplicating the + // GOEXPERIMENT-related build logic from cmd/go here. If the + // bootstrap toolchain is < Go 1.17, it will ignore this + // anyway since GOEXPERIMENT is baked in; otherwise it will + // pick it up from the environment we set here. Once we're + // using toolchain1 with dist as the build system, we need to + // override this to keep the experiments assumed by the + // toolchain and by dist consistent. Once go_bootstrap takes + // over the build process, we'll set this back to the original + // GOEXPERIMENT. + os.Setenv("GOEXPERIMENT", "none") + + if debug { + // cmd/buildid is used in debug mode. + toolchain = append(toolchain, "cmd/buildid") + } + + if isdir(pathf("%s/src/pkg", goroot)) { + fatalf("\n\n"+ + "The Go package sources have moved to $GOROOT/src.\n"+ + "*** %s still exists. ***\n"+ + "It probably contains stale files that may confuse the build.\n"+ + "Please (check what's there and) remove it and try again.\n"+ + "See https://golang.org/s/go14nopkg\n", + pathf("%s/src/pkg", goroot)) + } + + if rebuildall { + clean() + } + + setup() + + timelog("build", "toolchain1") + checkCC() + bootstrapBuildTools() + + // Remember old content of $GOROOT/bin for comparison below. + oldBinFiles, _ := filepath.Glob(pathf("%s/bin/*", goroot)) + + // For the main bootstrap, building for host os/arch. + oldgoos = goos + oldgoarch = goarch + goos = gohostos + goarch = gohostarch + os.Setenv("GOHOSTARCH", gohostarch) + os.Setenv("GOHOSTOS", gohostos) + os.Setenv("GOARCH", goarch) + os.Setenv("GOOS", goos) + + timelog("build", "go_bootstrap") + xprintf("Building Go bootstrap cmd/go (go_bootstrap) using Go toolchain1.\n") + install("runtime") // dependency not visible in sources; also sets up textflag.h + install("cmd/go") + if vflag > 0 { + xprintf("\n") + } + + gogcflags = os.Getenv("GO_GCFLAGS") // we were using $BOOT_GO_GCFLAGS until now + goldflags = os.Getenv("GO_LDFLAGS") // we were using $BOOT_GO_LDFLAGS until now + goBootstrap := pathf("%s/go_bootstrap", tooldir) + cmdGo := pathf("%s/go", gobin) + if debug { + run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full") + copyfile(pathf("%s/compile1", tooldir), pathf("%s/compile", tooldir), writeExec) + } + + // To recap, so far we have built the new toolchain + // (cmd/asm, cmd/cgo, cmd/compile, cmd/link) + // using Go 1.4's toolchain and go command. + // Then we built the new go command (as go_bootstrap) + // using the new toolchain and our own build logic (above). + // + // toolchain1 = mk(new toolchain, go1.4 toolchain, go1.4 cmd/go) + // go_bootstrap = mk(new cmd/go, toolchain1, cmd/dist) + // + // The toolchain1 we built earlier is built from the new sources, + // but because it was built using cmd/go it has no build IDs. + // The eventually installed toolchain needs build IDs, so we need + // to do another round: + // + // toolchain2 = mk(new toolchain, toolchain1, go_bootstrap) + // + timelog("build", "toolchain2") + if vflag > 0 { + xprintf("\n") + } + xprintf("Building Go toolchain2 using go_bootstrap and Go toolchain1.\n") + os.Setenv("CC", compilerEnvLookup(defaultcc, goos, goarch)) + // Now that cmd/go is in charge of the build process, enable GOEXPERIMENT. + os.Setenv("GOEXPERIMENT", goexperiment) + goInstall(goBootstrap, append([]string{"-i"}, toolchain...)...) + if debug { + run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full") + run("", ShowOutput|CheckExit, pathf("%s/buildid", tooldir), pathf("%s/pkg/%s_%s/runtime/internal/sys.a", goroot, goos, goarch)) + copyfile(pathf("%s/compile2", tooldir), pathf("%s/compile", tooldir), writeExec) + } + + // Toolchain2 should be semantically equivalent to toolchain1, + // but it was built using the new compilers instead of the Go 1.4 compilers, + // so it should at the least run faster. Also, toolchain1 had no build IDs + // in the binaries, while toolchain2 does. In non-release builds, the + // toolchain's build IDs feed into constructing the build IDs of built targets, + // so in non-release builds, everything now looks out-of-date due to + // toolchain2 having build IDs - that is, due to the go command seeing + // that there are new compilers. In release builds, the toolchain's reported + // version is used in place of the build ID, and the go command does not + // see that change from toolchain1 to toolchain2, so in release builds, + // nothing looks out of date. + // To keep the behavior the same in both non-release and release builds, + // we force-install everything here. + // + // toolchain3 = mk(new toolchain, toolchain2, go_bootstrap) + // + timelog("build", "toolchain3") + if vflag > 0 { + xprintf("\n") + } + xprintf("Building Go toolchain3 using go_bootstrap and Go toolchain2.\n") + goInstall(goBootstrap, append([]string{"-a", "-i"}, toolchain...)...) + if debug { + run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full") + run("", ShowOutput|CheckExit, pathf("%s/buildid", tooldir), pathf("%s/pkg/%s_%s/runtime/internal/sys.a", goroot, goos, goarch)) + copyfile(pathf("%s/compile3", tooldir), pathf("%s/compile", tooldir), writeExec) + } + checkNotStale(goBootstrap, append(toolchain, "runtime/internal/sys")...) + + if goos == oldgoos && goarch == oldgoarch { + // Common case - not setting up for cross-compilation. + timelog("build", "toolchain") + if vflag > 0 { + xprintf("\n") + } + xprintf("Building packages and commands for %s/%s.\n", goos, goarch) + } else { + // GOOS/GOARCH does not match GOHOSTOS/GOHOSTARCH. + // Finish GOHOSTOS/GOHOSTARCH installation and then + // run GOOS/GOARCH installation. + timelog("build", "host toolchain") + if vflag > 0 { + xprintf("\n") + } + xprintf("Building packages and commands for host, %s/%s.\n", goos, goarch) + goInstall(goBootstrap, "std", "cmd") + checkNotStale(goBootstrap, "std", "cmd") + checkNotStale(cmdGo, "std", "cmd") + + timelog("build", "target toolchain") + if vflag > 0 { + xprintf("\n") + } + goos = oldgoos + goarch = oldgoarch + os.Setenv("GOOS", goos) + os.Setenv("GOARCH", goarch) + os.Setenv("CC", compilerEnvLookup(defaultcc, goos, goarch)) + xprintf("Building packages and commands for target, %s/%s.\n", goos, goarch) + } + targets := []string{"std", "cmd"} + if goos == "js" && goarch == "wasm" { + // Skip the cmd tools for js/wasm. They're not usable. + targets = targets[:1] + } + goInstall(goBootstrap, targets...) + checkNotStale(goBootstrap, targets...) + checkNotStale(cmdGo, targets...) + if debug { + run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full") + run("", ShowOutput|CheckExit, pathf("%s/buildid", tooldir), pathf("%s/pkg/%s_%s/runtime/internal/sys.a", goroot, goos, goarch)) + checkNotStale(goBootstrap, append(toolchain, "runtime/internal/sys")...) + copyfile(pathf("%s/compile4", tooldir), pathf("%s/compile", tooldir), writeExec) + } + + // Check that there are no new files in $GOROOT/bin other than + // go and gofmt and $GOOS_$GOARCH (target bin when cross-compiling). + binFiles, _ := filepath.Glob(pathf("%s/bin/*", goroot)) + ok := map[string]bool{} + for _, f := range oldBinFiles { + ok[f] = true + } + for _, f := range binFiles { + elem := strings.TrimSuffix(filepath.Base(f), ".exe") + if !ok[f] && elem != "go" && elem != "gofmt" && elem != goos+"_"+goarch { + fatalf("unexpected new file in $GOROOT/bin: %s", elem) + } + } + + // Remove go_bootstrap now that we're done. + xremove(pathf("%s/go_bootstrap", tooldir)) + + if goos == "android" { + // Make sure the exec wrapper will sync a fresh $GOROOT to the device. + xremove(pathf("%s/go_android_exec-adb-sync-status", os.TempDir())) + } + + if wrapperPath := wrapperPathFor(goos, goarch); wrapperPath != "" { + oldcc := os.Getenv("CC") + os.Setenv("GOOS", gohostos) + os.Setenv("GOARCH", gohostarch) + os.Setenv("CC", compilerEnvLookup(defaultcc, gohostos, gohostarch)) + goCmd(cmdGo, "build", "-o", pathf("%s/go_%s_%s_exec%s", gobin, goos, goarch, exe), wrapperPath) + // Restore environment. + // TODO(elias.naur): support environment variables in goCmd? + os.Setenv("GOOS", goos) + os.Setenv("GOARCH", goarch) + os.Setenv("CC", oldcc) + } + + // Print trailing banner unless instructed otherwise. + if !noBanner { + banner() + } +} + +func wrapperPathFor(goos, goarch string) string { + switch { + case goos == "android": + if gohostos != "android" { + return pathf("%s/misc/android/go_android_exec.go", goroot) + } + case goos == "ios": + if gohostos != "ios" { + return pathf("%s/misc/ios/go_ios_exec.go", goroot) + } + } + return "" +} + +func goInstall(goBinary string, args ...string) { + goCmd(goBinary, "install", args...) +} + +func goCmd(goBinary string, cmd string, args ...string) { + goCmd := []string{goBinary, cmd, "-gcflags=all=" + gogcflags, "-ldflags=all=" + goldflags} + if vflag > 0 { + goCmd = append(goCmd, "-v") + } + + // Force only one process at a time on vx32 emulation. + if gohostos == "plan9" && os.Getenv("sysname") == "vx32" { + goCmd = append(goCmd, "-p=1") + } + + run(workdir, ShowOutput|CheckExit, append(goCmd, args...)...) +} + +func checkNotStale(goBinary string, targets ...string) { + out := run(workdir, CheckExit, + append([]string{ + goBinary, + "list", "-gcflags=all=" + gogcflags, "-ldflags=all=" + goldflags, + "-f={{if .Stale}}\tSTALE {{.ImportPath}}: {{.StaleReason}}{{end}}", + }, targets...)...) + if strings.Contains(out, "\tSTALE ") { + os.Setenv("GODEBUG", "gocachehash=1") + for _, target := range []string{"runtime/internal/sys", "cmd/dist", "cmd/link"} { + if strings.Contains(out, "STALE "+target) { + run(workdir, ShowOutput|CheckExit, goBinary, "list", "-f={{.ImportPath}} {{.Stale}}", target) + break + } + } + fatalf("unexpected stale targets reported by %s list -gcflags=\"%s\" -ldflags=\"%s\" for %v (consider rerunning with GOMAXPROCS=1 GODEBUG=gocachehash=1):\n%s", goBinary, gogcflags, goldflags, targets, out) + } +} + +// Cannot use go/build directly because cmd/dist for a new release +// builds against an old release's go/build, which may be out of sync. +// To reduce duplication, we generate the list for go/build from this. +// +// We list all supported platforms in this list, so that this is the +// single point of truth for supported platforms. This list is used +// by 'go tool dist list'. +var cgoEnabled = map[string]bool{ + "aix/ppc64": true, + "darwin/amd64": true, + "darwin/arm64": true, + "dragonfly/amd64": true, + "freebsd/386": true, + "freebsd/amd64": true, + "freebsd/arm": true, + "freebsd/arm64": true, + "illumos/amd64": true, + "linux/386": true, + "linux/amd64": true, + "linux/arm": true, + "linux/arm64": true, + "linux/ppc64": false, + "linux/ppc64le": true, + "linux/mips": true, + "linux/mipsle": true, + "linux/mips64": true, + "linux/mips64le": true, + "linux/riscv64": true, + "linux/s390x": true, + "linux/sparc64": true, + "android/386": true, + "android/amd64": true, + "android/arm": true, + "android/arm64": true, + "ios/arm64": true, + "ios/amd64": true, + "js/wasm": false, + "netbsd/386": true, + "netbsd/amd64": true, + "netbsd/arm": true, + "netbsd/arm64": true, + "openbsd/386": true, + "openbsd/amd64": true, + "openbsd/arm": true, + "openbsd/arm64": true, + "openbsd/mips64": true, + "plan9/386": false, + "plan9/amd64": false, + "plan9/arm": false, + "solaris/amd64": true, + "windows/386": true, + "windows/amd64": true, + "windows/arm": false, + "windows/arm64": true, +} + +// List of platforms which are supported but not complete yet. These get +// filtered out of cgoEnabled for 'dist list'. See golang.org/issue/28944 +var incomplete = map[string]bool{ + "linux/sparc64": true, +} + +// List of platforms which are first class ports. See golang.org/issue/38874. +var firstClass = map[string]bool{ + "darwin/amd64": true, + "darwin/arm64": true, + "linux/386": true, + "linux/amd64": true, + "linux/arm": true, + "linux/arm64": true, + "windows/386": true, + "windows/amd64": true, +} + +func needCC() bool { + switch os.Getenv("CGO_ENABLED") { + case "1": + return true + case "0": + return false + } + return cgoEnabled[gohostos+"/"+gohostarch] +} + +func checkCC() { + if !needCC() { + return + } + if output, err := exec.Command(defaultcc[""], "--help").CombinedOutput(); err != nil { + outputHdr := "" + if len(output) > 0 { + outputHdr = "\nCommand output:\n\n" + } + fatalf("cannot invoke C compiler %q: %v\n\n"+ + "Go needs a system C compiler for use with cgo.\n"+ + "To set a C compiler, set CC=the-compiler.\n"+ + "To disable cgo, set CGO_ENABLED=0.\n%s%s", defaultcc[""], err, outputHdr, output) + } +} + +func defaulttarg() string { + // xgetwd might return a path with symlinks fully resolved, and if + // there happens to be symlinks in goroot, then the hasprefix test + // will never succeed. Instead, we use xrealwd to get a canonical + // goroot/src before the comparison to avoid this problem. + pwd := xgetwd() + src := pathf("%s/src/", goroot) + real_src := xrealwd(src) + if !strings.HasPrefix(pwd, real_src) { + fatalf("current directory %s is not under %s", pwd, real_src) + } + pwd = pwd[len(real_src):] + // guard against xrealwd returning the directory without the trailing / + pwd = strings.TrimPrefix(pwd, "/") + + return pwd +} + +// Install installs the list of packages named on the command line. +func cmdinstall() { + xflagparse(-1) + + if flag.NArg() == 0 { + install(defaulttarg()) + } + + for _, arg := range flag.Args() { + install(arg) + } +} + +// Clean deletes temporary objects. +func cmdclean() { + xflagparse(0) + clean() +} + +// Banner prints the 'now you've installed Go' banner. +func cmdbanner() { + xflagparse(0) + banner() +} + +func banner() { + if vflag > 0 { + xprintf("\n") + } + xprintf("---\n") + xprintf("Installed Go for %s/%s in %s\n", goos, goarch, goroot) + xprintf("Installed commands in %s\n", gobin) + + if !xsamefile(goroot_final, goroot) { + // If the files are to be moved, don't check that gobin + // is on PATH; assume they know what they are doing. + } else if gohostos == "plan9" { + // Check that gobin is bound before /bin. + pid := strings.Replace(readfile("#c/pid"), " ", "", -1) + ns := fmt.Sprintf("/proc/%s/ns", pid) + if !strings.Contains(readfile(ns), fmt.Sprintf("bind -b %s /bin", gobin)) { + xprintf("*** You need to bind %s before /bin.\n", gobin) + } + } else { + // Check that gobin appears in $PATH. + pathsep := ":" + if gohostos == "windows" { + pathsep = ";" + } + if !strings.Contains(pathsep+os.Getenv("PATH")+pathsep, pathsep+gobin+pathsep) { + xprintf("*** You need to add %s to your PATH.\n", gobin) + } + } + + if !xsamefile(goroot_final, goroot) { + xprintf("\n"+ + "The binaries expect %s to be copied or moved to %s\n", + goroot, goroot_final) + } +} + +// Version prints the Go version. +func cmdversion() { + xflagparse(0) + xprintf("%s\n", findgoversion()) +} + +// cmdlist lists all supported platforms. +func cmdlist() { + jsonFlag := flag.Bool("json", false, "produce JSON output") + xflagparse(0) + + var plats []string + for p := range cgoEnabled { + if incomplete[p] { + continue + } + plats = append(plats, p) + } + sort.Strings(plats) + + if !*jsonFlag { + for _, p := range plats { + xprintf("%s\n", p) + } + return + } + + type jsonResult struct { + GOOS string + GOARCH string + CgoSupported bool + FirstClass bool + } + var results []jsonResult + for _, p := range plats { + fields := strings.Split(p, "/") + results = append(results, jsonResult{ + GOOS: fields[0], + GOARCH: fields[1], + CgoSupported: cgoEnabled[p], + FirstClass: firstClass[p]}) + } + out, err := json.MarshalIndent(results, "", "\t") + if err != nil { + fatalf("json marshal error: %v", err) + } + if _, err := os.Stdout.Write(out); err != nil { + fatalf("write failed: %v", err) + } +} + +// IsRuntimePackagePath examines 'pkgpath' and returns TRUE if it +// belongs to the collection of "runtime-related" packages, including +// "runtime" itself, "reflect", "syscall", and the +// "runtime/internal/*" packages. +// +// Keep in sync with cmd/internal/objabi/path.go:IsRuntimePackagePath. +func IsRuntimePackagePath(pkgpath string) bool { + rval := false + switch pkgpath { + case "runtime": + rval = true + case "reflect": + rval = true + case "syscall": + rval = true + case "internal/bytealg": + rval = true + default: + rval = strings.HasPrefix(pkgpath, "runtime/internal") + } + return rval +} diff --git a/src/cmd/dist/buildgo.go b/src/cmd/dist/buildgo.go new file mode 100644 index 0000000..caafc13 --- /dev/null +++ b/src/cmd/dist/buildgo.go @@ -0,0 +1,127 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +/* + * Helpers for building cmd/go and cmd/cgo. + */ + +// mkzdefaultcc writes zdefaultcc.go: +// +// package main +// const defaultCC = <defaultcc> +// const defaultCXX = <defaultcxx> +// const defaultPkgConfig = <defaultpkgconfig> +// +// It is invoked to write cmd/go/internal/cfg/zdefaultcc.go +// but we also write cmd/cgo/zdefaultcc.go +func mkzdefaultcc(dir, file string) { + if strings.Contains(file, filepath.FromSlash("go/internal/cfg")) { + var buf bytes.Buffer + fmt.Fprintf(&buf, "// Code generated by go tool dist; DO NOT EDIT.\n") + fmt.Fprintln(&buf) + fmt.Fprintf(&buf, "package cfg\n") + fmt.Fprintln(&buf) + fmt.Fprintf(&buf, "const DefaultPkgConfig = `%s`\n", defaultpkgconfig) + buf.WriteString(defaultCCFunc("DefaultCC", defaultcc)) + buf.WriteString(defaultCCFunc("DefaultCXX", defaultcxx)) + writefile(buf.String(), file, writeSkipSame) + return + } + + var buf bytes.Buffer + fmt.Fprintf(&buf, "// Code generated by go tool dist; DO NOT EDIT.\n") + fmt.Fprintln(&buf) + fmt.Fprintf(&buf, "package main\n") + fmt.Fprintln(&buf) + fmt.Fprintf(&buf, "const defaultPkgConfig = `%s`\n", defaultpkgconfig) + buf.WriteString(defaultCCFunc("defaultCC", defaultcc)) + buf.WriteString(defaultCCFunc("defaultCXX", defaultcxx)) + writefile(buf.String(), file, writeSkipSame) +} + +func defaultCCFunc(name string, defaultcc map[string]string) string { + var buf bytes.Buffer + + fmt.Fprintf(&buf, "func %s(goos, goarch string) string {\n", name) + fmt.Fprintf(&buf, "\tswitch goos+`/`+goarch {\n") + var keys []string + for k := range defaultcc { + if k != "" { + keys = append(keys, k) + } + } + sort.Strings(keys) + for _, k := range keys { + fmt.Fprintf(&buf, "\tcase %q:\n\t\treturn %q\n", k, defaultcc[k]) + } + fmt.Fprintf(&buf, "\t}\n") + fmt.Fprintf(&buf, "\treturn %q\n", defaultcc[""]) + fmt.Fprintf(&buf, "}\n") + + return buf.String() +} + +// mkzcgo writes zosarch.go for cmd/go. +func mkzosarch(dir, file string) { + // sort for deterministic zosarch.go file + var list []string + for plat := range cgoEnabled { + list = append(list, plat) + } + sort.Strings(list) + + var buf bytes.Buffer + fmt.Fprintf(&buf, "// Code generated by go tool dist; DO NOT EDIT.\n\n") + fmt.Fprintf(&buf, "package cfg\n\n") + fmt.Fprintf(&buf, "var OSArchSupportsCgo = map[string]bool{\n") + for _, plat := range list { + fmt.Fprintf(&buf, "\t%q: %v,\n", plat, cgoEnabled[plat]) + } + fmt.Fprintf(&buf, "}\n") + + writefile(buf.String(), file, writeSkipSame) +} + +// mkzcgo writes zcgo.go for the go/build package: +// +// package build +// var cgoEnabled = map[string]bool{} +// +// It is invoked to write go/build/zcgo.go. +func mkzcgo(dir, file string) { + // sort for deterministic zcgo.go file + var list []string + for plat, hasCgo := range cgoEnabled { + if hasCgo { + list = append(list, plat) + } + } + sort.Strings(list) + + var buf bytes.Buffer + fmt.Fprintf(&buf, "// Code generated by go tool dist; DO NOT EDIT.\n") + fmt.Fprintln(&buf) + fmt.Fprintf(&buf, "package build\n") + fmt.Fprintln(&buf) + fmt.Fprintf(&buf, "const defaultCGO_ENABLED = %q\n", os.Getenv("CGO_ENABLED")) + fmt.Fprintln(&buf) + fmt.Fprintf(&buf, "var cgoEnabled = map[string]bool{\n") + for _, plat := range list { + fmt.Fprintf(&buf, "\t%q: true,\n", plat) + } + fmt.Fprintf(&buf, "}\n") + + writefile(buf.String(), file, writeSkipSame) +} diff --git a/src/cmd/dist/buildruntime.go b/src/cmd/dist/buildruntime.go new file mode 100644 index 0000000..fdc1d25 --- /dev/null +++ b/src/cmd/dist/buildruntime.go @@ -0,0 +1,105 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "fmt" + "os" + "strings" +) + +/* + * Helpers for building runtime. + */ + +// mkzversion writes zversion.go: +// +// package sys +// +// const StackGuardMultiplier = <multiplier value> +// +func mkzversion(dir, file string) { + var buf bytes.Buffer + fmt.Fprintf(&buf, "// Code generated by go tool dist; DO NOT EDIT.\n") + fmt.Fprintln(&buf) + fmt.Fprintf(&buf, "package sys\n") + fmt.Fprintln(&buf) + fmt.Fprintf(&buf, "const StackGuardMultiplierDefault = %d\n", stackGuardMultiplierDefault()) + + writefile(buf.String(), file, writeSkipSame) +} + +// mkbuildcfg writes internal/buildcfg/zbootstrap.go: +// +// package buildcfg +// +// const defaultGOROOT = <goroot> +// const defaultGO386 = <go386> +// ... +// const defaultGOOS = runtime.GOOS +// const defaultGOARCH = runtime.GOARCH +// +// The use of runtime.GOOS and runtime.GOARCH makes sure that +// a cross-compiled compiler expects to compile for its own target +// system. That is, if on a Mac you do: +// +// GOOS=linux GOARCH=ppc64 go build cmd/compile +// +// the resulting compiler will default to generating linux/ppc64 object files. +// This is more useful than having it default to generating objects for the +// original target (in this example, a Mac). +func mkbuildcfg(file string) { + var buf bytes.Buffer + fmt.Fprintf(&buf, "// Code generated by go tool dist; DO NOT EDIT.\n") + fmt.Fprintln(&buf) + fmt.Fprintf(&buf, "package buildcfg\n") + fmt.Fprintln(&buf) + fmt.Fprintf(&buf, "import \"runtime\"\n") + fmt.Fprintln(&buf) + fmt.Fprintf(&buf, "const defaultGO386 = `%s`\n", go386) + fmt.Fprintf(&buf, "const defaultGOAMD64 = `%s`\n", goamd64) + fmt.Fprintf(&buf, "const defaultGOARM = `%s`\n", goarm) + fmt.Fprintf(&buf, "const defaultGOMIPS = `%s`\n", gomips) + fmt.Fprintf(&buf, "const defaultGOMIPS64 = `%s`\n", gomips64) + fmt.Fprintf(&buf, "const defaultGOPPC64 = `%s`\n", goppc64) + fmt.Fprintf(&buf, "const defaultGOEXPERIMENT = `%s`\n", goexperiment) + fmt.Fprintf(&buf, "const defaultGO_EXTLINK_ENABLED = `%s`\n", goextlinkenabled) + fmt.Fprintf(&buf, "const defaultGO_LDSO = `%s`\n", defaultldso) + fmt.Fprintf(&buf, "const version = `%s`\n", findgoversion()) + fmt.Fprintf(&buf, "const defaultGOOS = runtime.GOOS\n") + fmt.Fprintf(&buf, "const defaultGOARCH = runtime.GOARCH\n") + + writefile(buf.String(), file, writeSkipSame) +} + +// mkobjabi writes cmd/internal/objabi/zbootstrap.go: +// +// package objabi +// +// const stackGuardMultiplierDefault = <multiplier value> +// +func mkobjabi(file string) { + var buf bytes.Buffer + fmt.Fprintf(&buf, "// Code generated by go tool dist; DO NOT EDIT.\n") + fmt.Fprintln(&buf) + fmt.Fprintf(&buf, "package objabi\n") + fmt.Fprintln(&buf) + fmt.Fprintf(&buf, "const stackGuardMultiplierDefault = %d\n", stackGuardMultiplierDefault()) + + writefile(buf.String(), file, writeSkipSame) +} + +// stackGuardMultiplierDefault returns a multiplier to apply to the default +// stack guard size. Larger multipliers are used for non-optimized +// builds that have larger stack frames. +func stackGuardMultiplierDefault() int { + for _, s := range strings.Split(os.Getenv("GO_GCFLAGS"), " ") { + if s == "-N" { + return 2 + } + } + return 1 +} diff --git a/src/cmd/dist/buildtag.go b/src/cmd/dist/buildtag.go new file mode 100644 index 0000000..24776a0 --- /dev/null +++ b/src/cmd/dist/buildtag.go @@ -0,0 +1,133 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "strings" +) + +// exprParser is a //go:build expression parser and evaluator. +// The parser is a trivial precedence-based parser which is still +// almost overkill for these very simple expressions. +type exprParser struct { + x string + t exprToken // upcoming token +} + +// val is the value type result of parsing. +// We don't keep a parse tree, just the value of the expression. +type val bool + +// exprToken describes a single token in the input. +// Prefix operators define a prefix func that parses the +// upcoming value. Binary operators define an infix func +// that combines two values according to the operator. +// In that case, the parsing loop parses the two values. +type exprToken struct { + tok string + prec int + prefix func(*exprParser) val + infix func(val, val) val +} + +var exprTokens []exprToken + +func init() { // init to break init cycle + exprTokens = []exprToken{ + {tok: "&&", prec: 1, infix: func(x, y val) val { return x && y }}, + {tok: "||", prec: 2, infix: func(x, y val) val { return x || y }}, + {tok: "!", prec: 3, prefix: (*exprParser).not}, + {tok: "(", prec: 3, prefix: (*exprParser).paren}, + {tok: ")"}, + } +} + +// matchexpr parses and evaluates the //go:build expression x. +func matchexpr(x string) (matched bool, err error) { + defer func() { + if e := recover(); e != nil { + matched = false + err = fmt.Errorf("parsing //go:build line: %v", e) + } + }() + + p := &exprParser{x: x} + p.next() + v := p.parse(0) + if p.t.tok != "end of expression" { + panic("unexpected " + p.t.tok) + } + return bool(v), nil +} + +// parse parses an expression, including binary operators at precedence >= prec. +func (p *exprParser) parse(prec int) val { + if p.t.prefix == nil { + panic("unexpected " + p.t.tok) + } + v := p.t.prefix(p) + for p.t.prec >= prec && p.t.infix != nil { + t := p.t + p.next() + v = t.infix(v, p.parse(t.prec+1)) + } + return v +} + +// not is the prefix parser for a ! token. +func (p *exprParser) not() val { + p.next() + return !p.parse(100) +} + +// paren is the prefix parser for a ( token. +func (p *exprParser) paren() val { + p.next() + v := p.parse(0) + if p.t.tok != ")" { + panic("missing )") + } + p.next() + return v +} + +// next advances the parser to the next token, +// leaving the token in p.t. +func (p *exprParser) next() { + p.x = strings.TrimSpace(p.x) + if p.x == "" { + p.t = exprToken{tok: "end of expression"} + return + } + for _, t := range exprTokens { + if strings.HasPrefix(p.x, t.tok) { + p.x = p.x[len(t.tok):] + p.t = t + return + } + } + + i := 0 + for i < len(p.x) && validtag(p.x[i]) { + i++ + } + if i == 0 { + panic(fmt.Sprintf("syntax error near %#q", rune(p.x[i]))) + } + tag := p.x[:i] + p.x = p.x[i:] + p.t = exprToken{ + tok: "tag", + prefix: func(p *exprParser) val { + p.next() + return val(matchtag(tag)) + }, + } +} + +func validtag(c byte) bool { + return 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || c == '.' || c == '_' +} diff --git a/src/cmd/dist/buildtag_test.go b/src/cmd/dist/buildtag_test.go new file mode 100644 index 0000000..f64abfd --- /dev/null +++ b/src/cmd/dist/buildtag_test.go @@ -0,0 +1,43 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "reflect" + "testing" +) + +var buildParserTests = []struct { + x string + matched bool + err error +}{ + {"gc", true, nil}, + {"gccgo", false, nil}, + {"!gc", false, nil}, + {"gc && gccgo", false, nil}, + {"gc || gccgo", true, nil}, + {"gc || (gccgo && !gccgo)", true, nil}, + {"gc && (gccgo || !gccgo)", true, nil}, + {"!(gc && (gccgo || !gccgo))", false, nil}, + {"gccgo || gc", true, nil}, + {"!(!(!(gccgo || gc)))", false, nil}, + {"compiler_bootstrap", false, nil}, + {"cmd_go_bootstrap", true, nil}, + {"syntax(error", false, fmt.Errorf("parsing //go:build line: unexpected (")}, + {"(gc", false, fmt.Errorf("parsing //go:build line: missing )")}, + {"gc gc", false, fmt.Errorf("parsing //go:build line: unexpected tag")}, + {"(gc))", false, fmt.Errorf("parsing //go:build line: unexpected )")}, +} + +func TestBuildParser(t *testing.T) { + for _, tt := range buildParserTests { + matched, err := matchexpr(tt.x) + if matched != tt.matched || !reflect.DeepEqual(err, tt.err) { + t.Errorf("matchexpr(%q) = %v, %v; want %v, %v", tt.x, matched, err, tt.matched, tt.err) + } + } +} diff --git a/src/cmd/dist/buildtool.go b/src/cmd/dist/buildtool.go new file mode 100644 index 0000000..036f8c5 --- /dev/null +++ b/src/cmd/dist/buildtool.go @@ -0,0 +1,325 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Build toolchain using Go 1.4. +// +// The general strategy is to copy the source files we need into +// a new GOPATH workspace, adjust import paths appropriately, +// invoke the Go 1.4 go command to build those sources, +// and then copy the binaries back. + +package main + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" +) + +// bootstrapDirs is a list of directories holding code that must be +// compiled with a Go 1.4 toolchain to produce the bootstrapTargets. +// All directories in this list are relative to and must be below $GOROOT/src. +// +// The list has two kinds of entries: names beginning with cmd/ with +// no other slashes, which are commands, and other paths, which are packages +// supporting the commands. Packages in the standard library can be listed +// if a newer copy needs to be substituted for the Go 1.4 copy when used +// by the command packages. Paths ending with /... automatically +// include all packages within subdirectories as well. +// These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big. +var bootstrapDirs = []string{ + "cmd/asm", + "cmd/asm/internal/...", + "cmd/cgo", + "cmd/compile", + "cmd/compile/internal/...", + "cmd/internal/archive", + "cmd/internal/bio", + "cmd/internal/codesign", + "cmd/internal/dwarf", + "cmd/internal/edit", + "cmd/internal/gcprog", + "cmd/internal/goobj", + "cmd/internal/obj/...", + "cmd/internal/objabi", + "cmd/internal/pkgpath", + "cmd/internal/quoted", + "cmd/internal/src", + "cmd/internal/sys", + "cmd/link", + "cmd/link/internal/...", + "compress/flate", + "compress/zlib", + "container/heap", + "debug/dwarf", + "debug/elf", + "debug/macho", + "debug/pe", + "go/constant", + "internal/buildcfg", + "internal/goexperiment", + "internal/goversion", + "internal/race", + "internal/unsafeheader", + "internal/xcoff", + "math/big", + "math/bits", + "sort", + "strconv", +} + +// File prefixes that are ignored by go/build anyway, and cause +// problems with editor generated temporary files (#18931). +var ignorePrefixes = []string{ + ".", + "_", + "#", +} + +// File suffixes that use build tags introduced since Go 1.4. +// These must not be copied into the bootstrap build directory. +// Also ignore test files. +var ignoreSuffixes = []string{ + "_arm64.s", + "_arm64.go", + "_riscv64.s", + "_riscv64.go", + "_wasm.s", + "_wasm.go", + "_test.s", + "_test.go", +} + +var tryDirs = []string{ + "sdk/go1.17", + "go1.17", +} + +func bootstrapBuildTools() { + goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP") + if goroot_bootstrap == "" { + home := os.Getenv("HOME") + goroot_bootstrap = pathf("%s/go1.4", home) + for _, d := range tryDirs { + if p := pathf("%s/%s", home, d); isdir(p) { + goroot_bootstrap = p + } + } + } + xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap) + + mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot)) + mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot)) + + // Use $GOROOT/pkg/bootstrap as the bootstrap workspace root. + // We use a subdirectory of $GOROOT/pkg because that's the + // space within $GOROOT where we store all generated objects. + // We could use a temporary directory outside $GOROOT instead, + // but it is easier to debug on failure if the files are in a known location. + workspace := pathf("%s/pkg/bootstrap", goroot) + xremoveall(workspace) + xatexit(func() { xremoveall(workspace) }) + base := pathf("%s/src/bootstrap", workspace) + xmkdirall(base) + + // Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths. + writefile("module bootstrap\n", pathf("%s/%s", base, "go.mod"), 0) + for _, dir := range bootstrapDirs { + recurse := strings.HasSuffix(dir, "/...") + dir = strings.TrimSuffix(dir, "/...") + filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + fatalf("walking bootstrap dirs failed: %v: %v", path, err) + } + + name := filepath.Base(path) + src := pathf("%s/src/%s", goroot, path) + dst := pathf("%s/%s", base, path) + + if info.IsDir() { + if !recurse && path != dir || name == "testdata" { + return filepath.SkipDir + } + + xmkdirall(dst) + if path == "cmd/cgo" { + // Write to src because we need the file both for bootstrap + // and for later in the main build. + mkzdefaultcc("", pathf("%s/zdefaultcc.go", src)) + mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst)) + } + return nil + } + + for _, pre := range ignorePrefixes { + if strings.HasPrefix(name, pre) { + return nil + } + } + for _, suf := range ignoreSuffixes { + if strings.HasSuffix(name, suf) { + return nil + } + } + + text := bootstrapRewriteFile(src) + writefile(text, dst, 0) + return nil + }) + } + + // Set up environment for invoking Go 1.4 go command. + // GOROOT points at Go 1.4 GOROOT, + // GOPATH points at our bootstrap workspace, + // GOBIN is empty, so that binaries are installed to GOPATH/bin, + // and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty, + // so that Go 1.4 builds whatever kind of binary it knows how to build. + // Restore GOROOT, GOPATH, and GOBIN when done. + // Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH, + // because setup will take care of those when bootstrapBuildTools returns. + + defer os.Setenv("GOROOT", os.Getenv("GOROOT")) + os.Setenv("GOROOT", goroot_bootstrap) + + defer os.Setenv("GOPATH", os.Getenv("GOPATH")) + os.Setenv("GOPATH", workspace) + + defer os.Setenv("GOBIN", os.Getenv("GOBIN")) + os.Setenv("GOBIN", "") + + os.Setenv("GOOS", "") + os.Setenv("GOHOSTOS", "") + os.Setenv("GOARCH", "") + os.Setenv("GOHOSTARCH", "") + + // Run Go 1.4 to build binaries. Use -gcflags=-l to disable inlining to + // workaround bugs in Go 1.4's compiler. See discussion thread: + // https://groups.google.com/d/msg/golang-dev/Ss7mCKsvk8w/Gsq7VYI0AwAJ + // Use the math_big_pure_go build tag to disable the assembly in math/big + // which may contain unsupported instructions. + // Note that if we are using Go 1.10 or later as bootstrap, the -gcflags=-l + // only applies to the final cmd/go binary, but that's OK: if this is Go 1.10 + // or later we don't need to disable inlining to work around bugs in the Go 1.4 compiler. + cmd := []string{ + pathf("%s/bin/go", goroot_bootstrap), + "install", + "-gcflags=-l", + "-tags=math_big_pure_go compiler_bootstrap", + } + if vflag > 0 { + cmd = append(cmd, "-v") + } + if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" { + cmd = append(cmd, "-toolexec="+tool) + } + cmd = append(cmd, "bootstrap/cmd/...") + run(base, ShowOutput|CheckExit, cmd...) + + // Copy binaries into tool binary directory. + for _, name := range bootstrapDirs { + if !strings.HasPrefix(name, "cmd/") { + continue + } + name = name[len("cmd/"):] + if !strings.Contains(name, "/") { + copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec) + } + } + + if vflag > 0 { + xprintf("\n") + } +} + +var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite") + +// isUnneededSSARewriteFile reports whether srcFile is a +// src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an +// architecture that isn't for the current runtime.GOARCH. +// +// When unneeded is true archCaps is the rewrite base filename without +// the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc. +func isUnneededSSARewriteFile(srcFile string) (archCaps string, unneeded bool) { + if !strings.Contains(srcFile, ssaRewriteFileSubstring) { + return "", false + } + fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go") + if fileArch == "" { + return "", false + } + b := fileArch[0] + if b == '_' || ('a' <= b && b <= 'z') { + return "", false + } + archCaps = fileArch + fileArch = strings.ToLower(fileArch) + fileArch = strings.TrimSuffix(fileArch, "splitload") + if fileArch == os.Getenv("GOHOSTARCH") { + return "", false + } + if fileArch == strings.TrimSuffix(runtime.GOARCH, "le") { + return "", false + } + if fileArch == strings.TrimSuffix(os.Getenv("GOARCH"), "le") { + return "", false + } + return archCaps, true +} + +func bootstrapRewriteFile(srcFile string) string { + // During bootstrap, generate dummy rewrite files for + // irrelevant architectures. We only need to build a bootstrap + // binary that works for the current runtime.GOARCH. + // This saves 6+ seconds of bootstrap. + if archCaps, ok := isUnneededSSARewriteFile(srcFile); ok { + return fmt.Sprintf(`// Code generated by go tool dist; DO NOT EDIT. + +package ssa + +func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") } +func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") } +`, archCaps, archCaps) + } + + return bootstrapFixImports(srcFile) +} + +func bootstrapFixImports(srcFile string) string { + text := readfile(srcFile) + if !strings.Contains(srcFile, "/cmd/") && !strings.Contains(srcFile, `\cmd\`) { + text = regexp.MustCompile(`\bany\b`).ReplaceAllString(text, "interface{}") + } + lines := strings.SplitAfter(text, "\n") + inBlock := false + for i, line := range lines { + if strings.HasPrefix(line, "import (") { + inBlock = true + continue + } + if inBlock && strings.HasPrefix(line, ")") { + inBlock = false + continue + } + if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) || + inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"") || strings.HasPrefix(line, "\texec \"")) { + line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1) + // During bootstrap, must use plain os/exec. + line = strings.Replace(line, `exec "internal/execabs"`, `"os/exec"`, -1) + for _, dir := range bootstrapDirs { + if strings.HasPrefix(dir, "cmd/") { + continue + } + line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1) + } + lines[i] = line + } + } + + lines[0] = "// Code generated by go tool dist; DO NOT EDIT.\n// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0] + + return strings.Join(lines, "") +} diff --git a/src/cmd/dist/doc.go b/src/cmd/dist/doc.go new file mode 100644 index 0000000..a4e6aa5 --- /dev/null +++ b/src/cmd/dist/doc.go @@ -0,0 +1,19 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Dist helps bootstrap, build, and test the Go distribution. +// +// Usage: +// go tool dist [command] +// +// The commands are: +// banner print installation banner +// bootstrap rebuild everything +// clean deletes all built files +// env [-p] print environment (-p: include $PATH) +// install [dir] install individual directory +// list [-json] list all supported platforms +// test [-h] run Go test(s) +// version print Go version +package main diff --git a/src/cmd/dist/exec.go b/src/cmd/dist/exec.go new file mode 100644 index 0000000..6730553 --- /dev/null +++ b/src/cmd/dist/exec.go @@ -0,0 +1,53 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "os/exec" + "strings" +) + +// setDir sets cmd.Dir to dir, and also adds PWD=dir to cmd's environment. +func setDir(cmd *exec.Cmd, dir string) { + cmd.Dir = dir + setEnv(cmd, "PWD", dir) +} + +// setEnv sets cmd.Env so that key = value. +// +// It first removes any existing values for key, so it is safe to call +// even from within cmdbootstrap. +func setEnv(cmd *exec.Cmd, key, value string) { + kv := key + "=" + value + if cmd.Env == nil { + cmd.Env = os.Environ() + } + + prefix := kv[:len(key)+1] + for i, entry := range cmd.Env { + if strings.HasPrefix(entry, prefix) { + cmd.Env[i] = kv + return + } + } + + cmd.Env = append(cmd.Env, kv) +} + +// unsetEnv sets cmd.Env so that key is not present in the environment. +func unsetEnv(cmd *exec.Cmd, key string) { + if cmd.Env == nil { + cmd.Env = os.Environ() + } + + prefix := key + "=" + for i, entry := range cmd.Env { + if strings.HasPrefix(entry, prefix) { + cmd.Env = append(cmd.Env[:i], cmd.Env[i+1:]...) + return + } + } +} diff --git a/src/cmd/dist/imports.go b/src/cmd/dist/imports.go new file mode 100644 index 0000000..05dd84d --- /dev/null +++ b/src/cmd/dist/imports.go @@ -0,0 +1,276 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file is forked from go/build/read.go. +// (cmd/dist must not import go/build because we do not want it to be +// sensitive to the specific version of go/build present in $GOROOT_BOOTSTRAP.) + +package main + +import ( + "bufio" + "errors" + "fmt" + "io" + "path" + "path/filepath" + "strconv" + "strings" + "unicode/utf8" +) + +type importReader struct { + b *bufio.Reader + buf []byte + peek byte + err error + eof bool + nerr int +} + +func isIdent(c byte) bool { + return 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || c == '_' || c >= utf8.RuneSelf +} + +var ( + errSyntax = errors.New("syntax error") + errNUL = errors.New("unexpected NUL in input") +) + +// syntaxError records a syntax error, but only if an I/O error has not already been recorded. +func (r *importReader) syntaxError() { + if r.err == nil { + r.err = errSyntax + } +} + +// readByte reads the next byte from the input, saves it in buf, and returns it. +// If an error occurs, readByte records the error in r.err and returns 0. +func (r *importReader) readByte() byte { + c, err := r.b.ReadByte() + if err == nil { + r.buf = append(r.buf, c) + if c == 0 { + err = errNUL + } + } + if err != nil { + if err == io.EOF { + r.eof = true + } else if r.err == nil { + r.err = err + } + c = 0 + } + return c +} + +// peekByte returns the next byte from the input reader but does not advance beyond it. +// If skipSpace is set, peekByte skips leading spaces and comments. +func (r *importReader) peekByte(skipSpace bool) byte { + if r.err != nil { + if r.nerr++; r.nerr > 10000 { + panic("go/build: import reader looping") + } + return 0 + } + + // Use r.peek as first input byte. + // Don't just return r.peek here: it might have been left by peekByte(false) + // and this might be peekByte(true). + c := r.peek + if c == 0 { + c = r.readByte() + } + for r.err == nil && !r.eof { + if skipSpace { + // For the purposes of this reader, semicolons are never necessary to + // understand the input and are treated as spaces. + switch c { + case ' ', '\f', '\t', '\r', '\n', ';': + c = r.readByte() + continue + + case '/': + c = r.readByte() + if c == '/' { + for c != '\n' && r.err == nil && !r.eof { + c = r.readByte() + } + } else if c == '*' { + var c1 byte + for (c != '*' || c1 != '/') && r.err == nil { + if r.eof { + r.syntaxError() + } + c, c1 = c1, r.readByte() + } + } else { + r.syntaxError() + } + c = r.readByte() + continue + } + } + break + } + r.peek = c + return r.peek +} + +// nextByte is like peekByte but advances beyond the returned byte. +func (r *importReader) nextByte(skipSpace bool) byte { + c := r.peekByte(skipSpace) + r.peek = 0 + return c +} + +// readKeyword reads the given keyword from the input. +// If the keyword is not present, readKeyword records a syntax error. +func (r *importReader) readKeyword(kw string) { + r.peekByte(true) + for i := 0; i < len(kw); i++ { + if r.nextByte(false) != kw[i] { + r.syntaxError() + return + } + } + if isIdent(r.peekByte(false)) { + r.syntaxError() + } +} + +// readIdent reads an identifier from the input. +// If an identifier is not present, readIdent records a syntax error. +func (r *importReader) readIdent() { + c := r.peekByte(true) + if !isIdent(c) { + r.syntaxError() + return + } + for isIdent(r.peekByte(false)) { + r.peek = 0 + } +} + +// readString reads a quoted string literal from the input. +// If an identifier is not present, readString records a syntax error. +func (r *importReader) readString(save *[]string) { + switch r.nextByte(true) { + case '`': + start := len(r.buf) - 1 + for r.err == nil { + if r.nextByte(false) == '`' { + if save != nil { + *save = append(*save, string(r.buf[start:])) + } + break + } + if r.eof { + r.syntaxError() + } + } + case '"': + start := len(r.buf) - 1 + for r.err == nil { + c := r.nextByte(false) + if c == '"' { + if save != nil { + *save = append(*save, string(r.buf[start:])) + } + break + } + if r.eof || c == '\n' { + r.syntaxError() + } + if c == '\\' { + r.nextByte(false) + } + } + default: + r.syntaxError() + } +} + +// readImport reads an import clause - optional identifier followed by quoted string - +// from the input. +func (r *importReader) readImport(imports *[]string) { + c := r.peekByte(true) + if c == '.' { + r.peek = 0 + } else if isIdent(c) { + r.readIdent() + } + r.readString(imports) +} + +// readComments is like ioutil.ReadAll, except that it only reads the leading +// block of comments in the file. +func readComments(f io.Reader) ([]byte, error) { + r := &importReader{b: bufio.NewReader(f)} + r.peekByte(true) + if r.err == nil && !r.eof { + // Didn't reach EOF, so must have found a non-space byte. Remove it. + r.buf = r.buf[:len(r.buf)-1] + } + return r.buf, r.err +} + +// readimports returns the imports found in the named file. +func readimports(file string) []string { + var imports []string + r := &importReader{b: bufio.NewReader(strings.NewReader(readfile(file)))} + r.readKeyword("package") + r.readIdent() + for r.peekByte(true) == 'i' { + r.readKeyword("import") + if r.peekByte(true) == '(' { + r.nextByte(false) + for r.peekByte(true) != ')' && r.err == nil { + r.readImport(&imports) + } + r.nextByte(false) + } else { + r.readImport(&imports) + } + } + + for i := range imports { + unquoted, err := strconv.Unquote(imports[i]) + if err != nil { + fatalf("reading imports from %s: %v", file, err) + } + imports[i] = unquoted + } + + return imports +} + +// resolveVendor returns a unique package path imported with the given import +// path from srcDir. +// +// resolveVendor assumes that a package is vendored if and only if its first +// path component contains a dot. If a package is vendored, its import path +// is returned with a "vendor" or "cmd/vendor" prefix, depending on srcDir. +// Otherwise, the import path is returned verbatim. +func resolveVendor(imp, srcDir string) string { + var first string + if i := strings.Index(imp, "/"); i < 0 { + first = imp + } else { + first = imp[:i] + } + isStandard := !strings.Contains(first, ".") + if isStandard { + return imp + } + + if strings.HasPrefix(srcDir, filepath.Join(goroot, "src", "cmd")) { + return path.Join("cmd", "vendor", imp) + } else if strings.HasPrefix(srcDir, filepath.Join(goroot, "src")) { + return path.Join("vendor", imp) + } else { + panic(fmt.Sprintf("srcDir %q not in GOOROT/src", srcDir)) + } +} diff --git a/src/cmd/dist/main.go b/src/cmd/dist/main.go new file mode 100644 index 0000000..212d5cb --- /dev/null +++ b/src/cmd/dist/main.go @@ -0,0 +1,189 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "flag" + "fmt" + "os" + "runtime" + "strings" +) + +func usage() { + xprintf(`usage: go tool dist [command] +Commands are: + +banner print installation banner +bootstrap rebuild everything +clean deletes all built files +env [-p] print environment (-p: include $PATH) +install [dir] install individual directory +list [-json] list all supported platforms +test [-h] run Go test(s) +version print Go version + +All commands take -v flags to emit extra information. +`) + xexit(2) +} + +// commands records the available commands. +var commands = map[string]func(){ + "banner": cmdbanner, + "bootstrap": cmdbootstrap, + "clean": cmdclean, + "env": cmdenv, + "install": cmdinstall, + "list": cmdlist, + "test": cmdtest, + "version": cmdversion, +} + +// main takes care of OS-specific startup and dispatches to xmain. +func main() { + os.Setenv("TERM", "dumb") // disable escape codes in clang errors + + // provide -check-armv6k first, before checking for $GOROOT so that + // it is possible to run this check without having $GOROOT available. + if len(os.Args) > 1 && os.Args[1] == "-check-armv6k" { + useARMv6K() // might fail with SIGILL + println("ARMv6K supported.") + os.Exit(0) + } + + gohostos = runtime.GOOS + switch gohostos { + case "aix": + // uname -m doesn't work under AIX + gohostarch = "ppc64" + case "darwin": + // macOS 10.9 and later require clang + defaultclang = true + case "freebsd": + // Since FreeBSD 10 gcc is no longer part of the base system. + defaultclang = true + case "openbsd": + // OpenBSD ships with GCC 4.2, which is now quite old. + defaultclang = true + case "plan9": + gohostarch = os.Getenv("objtype") + if gohostarch == "" { + fatalf("$objtype is unset") + } + case "solaris", "illumos": + // Solaris and illumos systems have multi-arch userlands, and + // "uname -m" reports the machine hardware name; e.g., + // "i86pc" on both 32- and 64-bit x86 systems. Check for the + // native (widest) instruction set on the running kernel: + out := run("", CheckExit, "isainfo", "-n") + if strings.Contains(out, "amd64") { + gohostarch = "amd64" + } + if strings.Contains(out, "i386") { + gohostarch = "386" + } + case "windows": + exe = ".exe" + } + + sysinit() + + if gohostarch == "" { + // Default Unix system. + out := run("", CheckExit, "uname", "-m") + outAll := run("", CheckExit, "uname", "-a") + switch { + case strings.Contains(outAll, "RELEASE_ARM64"): + // MacOS prints + // Darwin p1.local 21.1.0 Darwin Kernel Version 21.1.0: Wed Oct 13 17:33:01 PDT 2021; root:xnu-8019.41.5~1/RELEASE_ARM64_T6000 x86_64 + // on ARM64 laptops when there is an x86 parent in the + // process tree. Look for the RELEASE_ARM64 to avoid being + // confused into building an x86 toolchain. + gohostarch = "arm64" + case strings.Contains(out, "x86_64"), strings.Contains(out, "amd64"): + gohostarch = "amd64" + case strings.Contains(out, "86"): + gohostarch = "386" + if gohostos == "darwin" { + // Even on 64-bit platform, some versions of macOS uname -m prints i386. + // We don't support any of the OS X versions that run on 32-bit-only hardware anymore. + gohostarch = "amd64" + } + case strings.Contains(out, "aarch64"), strings.Contains(out, "arm64"): + gohostarch = "arm64" + case strings.Contains(out, "arm"): + gohostarch = "arm" + if gohostos == "netbsd" && strings.Contains(run("", CheckExit, "uname", "-p"), "aarch64") { + gohostarch = "arm64" + } + case strings.Contains(out, "ppc64le"): + gohostarch = "ppc64le" + case strings.Contains(out, "ppc64"): + gohostarch = "ppc64" + case strings.Contains(out, "mips64"): + gohostarch = "mips64" + if elfIsLittleEndian(os.Args[0]) { + gohostarch = "mips64le" + } + case strings.Contains(out, "mips"): + gohostarch = "mips" + if elfIsLittleEndian(os.Args[0]) { + gohostarch = "mipsle" + } + case strings.Contains(out, "riscv64"): + gohostarch = "riscv64" + case strings.Contains(out, "s390x"): + gohostarch = "s390x" + case gohostos == "darwin", gohostos == "ios": + if strings.Contains(run("", CheckExit, "uname", "-v"), "RELEASE_ARM64_") { + gohostarch = "arm64" + } + case gohostos == "openbsd": + if strings.Contains(run("", CheckExit, "uname", "-p"), "mips64") { + gohostarch = "mips64" + } + default: + fatalf("unknown architecture: %s", out) + } + } + + if gohostarch == "arm" || gohostarch == "mips64" || gohostarch == "mips64le" { + maxbg = min(maxbg, runtime.NumCPU()) + } + bginit() + + if len(os.Args) > 1 && os.Args[1] == "-check-goarm" { + useVFPv1() // might fail with SIGILL + println("VFPv1 OK.") + useVFPv3() // might fail with SIGILL + println("VFPv3 OK.") + os.Exit(0) + } + + xinit() + xmain() + xexit(0) +} + +// The OS-specific main calls into the portable code here. +func xmain() { + if len(os.Args) < 2 { + usage() + } + cmd := os.Args[1] + os.Args = os.Args[1:] // for flag parsing during cmd + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "usage: go tool dist %s [options]\n", cmd) + flag.PrintDefaults() + os.Exit(2) + } + if f, ok := commands[cmd]; ok { + f() + } else { + xprintf("unknown command %s\n", cmd) + usage() + } +} diff --git a/src/cmd/dist/sys_default.go b/src/cmd/dist/sys_default.go new file mode 100644 index 0000000..e87c84c --- /dev/null +++ b/src/cmd/dist/sys_default.go @@ -0,0 +1,11 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !windows +// +build !windows + +package main + +func sysinit() { +} diff --git a/src/cmd/dist/sys_windows.go b/src/cmd/dist/sys_windows.go new file mode 100644 index 0000000..265f729 --- /dev/null +++ b/src/cmd/dist/sys_windows.go @@ -0,0 +1,57 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "syscall" + "unsafe" +) + +var ( + modkernel32 = syscall.NewLazyDLL("kernel32.dll") + procGetSystemInfo = modkernel32.NewProc("GetSystemInfo") +) + +// see https://msdn.microsoft.com/en-us/library/windows/desktop/ms724958(v=vs.85).aspx +type systeminfo struct { + wProcessorArchitecture uint16 + wReserved uint16 + dwPageSize uint32 + lpMinimumApplicationAddress uintptr + lpMaximumApplicationAddress uintptr + dwActiveProcessorMask uintptr + dwNumberOfProcessors uint32 + dwProcessorType uint32 + dwAllocationGranularity uint32 + wProcessorLevel uint16 + wProcessorRevision uint16 +} + +// See https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/ns-sysinfoapi-system_info +const ( + PROCESSOR_ARCHITECTURE_AMD64 = 9 + PROCESSOR_ARCHITECTURE_INTEL = 0 + PROCESSOR_ARCHITECTURE_ARM = 5 + PROCESSOR_ARCHITECTURE_ARM64 = 12 + PROCESSOR_ARCHITECTURE_IA64 = 6 +) + +var sysinfo systeminfo + +func sysinit() { + syscall.Syscall(procGetSystemInfo.Addr(), 1, uintptr(unsafe.Pointer(&sysinfo)), 0, 0) + switch sysinfo.wProcessorArchitecture { + case PROCESSOR_ARCHITECTURE_AMD64: + gohostarch = "amd64" + case PROCESSOR_ARCHITECTURE_INTEL: + gohostarch = "386" + case PROCESSOR_ARCHITECTURE_ARM: + gohostarch = "arm" + case PROCESSOR_ARCHITECTURE_ARM64: + gohostarch = "arm64" + default: + fatalf("unknown processor architecture") + } +} diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go new file mode 100644 index 0000000..d9eb9c3 --- /dev/null +++ b/src/cmd/dist/test.go @@ -0,0 +1,1684 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strconv" + "strings" + "sync" + "time" +) + +func cmdtest() { + gogcflags = os.Getenv("GO_GCFLAGS") + + var t tester + var noRebuild bool + flag.BoolVar(&t.listMode, "list", false, "list available tests") + flag.BoolVar(&t.rebuild, "rebuild", false, "rebuild everything first") + flag.BoolVar(&noRebuild, "no-rebuild", false, "overrides -rebuild (historical dreg)") + flag.BoolVar(&t.keepGoing, "k", false, "keep going even when error occurred") + flag.BoolVar(&t.race, "race", false, "run in race builder mode (different set of tests)") + flag.BoolVar(&t.compileOnly, "compile-only", false, "compile tests, but don't run them. This is for some builders. Not all dist tests respect this flag, but most do.") + flag.StringVar(&t.banner, "banner", "##### ", "banner prefix; blank means no section banners") + flag.StringVar(&t.runRxStr, "run", os.Getenv("GOTESTONLY"), + "run only those tests matching the regular expression; empty means to run all. "+ + "Special exception: if the string begins with '!', the match is inverted.") + xflagparse(-1) // any number of args + if noRebuild { + t.rebuild = false + } + + t.run() +} + +// tester executes cmdtest. +type tester struct { + race bool + listMode bool + rebuild bool + failed bool + keepGoing bool + compileOnly bool // just try to compile all tests, but no need to run + runRxStr string + runRx *regexp.Regexp + runRxWant bool // want runRx to match (true) or not match (false) + runNames []string // tests to run, exclusive with runRx; empty means all + banner string // prefix, or "" for none + lastHeading string // last dir heading printed + + cgoEnabled bool + partial bool + haveTime bool // the 'time' binary is available + + tests []distTest + timeoutScale int + + worklist []*work +} + +type work struct { + dt *distTest + cmd *exec.Cmd + start chan bool + out []byte + err error + end chan bool +} + +// A distTest is a test run by dist test. +// Each test has a unique name and belongs to a group (heading) +type distTest struct { + name string // unique test name; may be filtered with -run flag + heading string // group section; this header is printed before the test is run. + fn func(*distTest) error +} + +func (t *tester) run() { + timelog("start", "dist test") + + var exeSuffix string + if goos == "windows" { + exeSuffix = ".exe" + } + if _, err := os.Stat(filepath.Join(gobin, "go"+exeSuffix)); err == nil { + os.Setenv("PATH", fmt.Sprintf("%s%c%s", gobin, os.PathListSeparator, os.Getenv("PATH"))) + } + + cmd := exec.Command("go", "env", "CGO_ENABLED") + cmd.Stderr = new(bytes.Buffer) + slurp, err := cmd.Output() + if err != nil { + fatalf("Error running go env CGO_ENABLED: %v\n%s", err, cmd.Stderr) + } + t.cgoEnabled, _ = strconv.ParseBool(strings.TrimSpace(string(slurp))) + if flag.NArg() > 0 && t.runRxStr != "" { + fatalf("the -run regular expression flag is mutually exclusive with test name arguments") + } + + t.runNames = flag.Args() + + if t.hasBash() { + if _, err := exec.LookPath("time"); err == nil { + t.haveTime = true + } + } + + // Set GOTRACEBACK to system if the user didn't set a level explicitly. + // Since we're running tests for Go, we want as much detail as possible + // if something goes wrong. + // + // Set it before running any commands just in case something goes wrong. + if ok := isEnvSet("GOTRACEBACK"); !ok { + if err := os.Setenv("GOTRACEBACK", "system"); err != nil { + if t.keepGoing { + log.Printf("Failed to set GOTRACEBACK: %v", err) + } else { + fatalf("Failed to set GOTRACEBACK: %v", err) + } + } + } + + if t.rebuild { + t.out("Building packages and commands.") + // Force rebuild the whole toolchain. + goInstall("go", append([]string{"-a", "-i"}, toolchain...)...) + } + + if !t.listMode { + if os.Getenv("GO_BUILDER_NAME") == "" { + // Complete rebuild bootstrap, even with -no-rebuild. + // If everything is up-to-date, this is a no-op. + // If everything is not up-to-date, the first checkNotStale + // during the test process will kill the tests, so we might + // as well install the world. + // Now that for example "go install cmd/compile" does not + // also install runtime (you need "go install -i cmd/compile" + // for that), it's easy for previous workflows like + // "rebuild the compiler and then run run.bash" + // to break if we don't automatically refresh things here. + // Rebuilding is a shortened bootstrap. + // See cmdbootstrap for a description of the overall process. + goInstall("go", append([]string{"-i"}, toolchain...)...) + goInstall("go", append([]string{"-i"}, toolchain...)...) + goInstall("go", "std", "cmd") + } else { + // The Go builder infrastructure should always begin running tests from a + // clean, non-stale state, so there is no need to rebuild the world. + // Instead, we can just check that it is not stale, which may be less + // expensive (and is also more likely to catch bugs in the builder + // implementation). + willTest := []string{"std"} + if t.shouldTestCmd() { + willTest = append(willTest, "cmd") + } + checkNotStale("go", willTest...) + } + } + + t.timeoutScale = 1 + switch goarch { + case "arm": + t.timeoutScale = 2 + case "mips", "mipsle", "mips64", "mips64le": + t.timeoutScale = 4 + } + if s := os.Getenv("GO_TEST_TIMEOUT_SCALE"); s != "" { + t.timeoutScale, err = strconv.Atoi(s) + if err != nil { + fatalf("failed to parse $GO_TEST_TIMEOUT_SCALE = %q as integer: %v", s, err) + } + } + + if t.runRxStr != "" { + if t.runRxStr[0] == '!' { + t.runRxWant = false + t.runRxStr = t.runRxStr[1:] + } else { + t.runRxWant = true + } + t.runRx = regexp.MustCompile(t.runRxStr) + } + + t.registerTests() + if t.listMode { + for _, tt := range t.tests { + fmt.Println(tt.name) + } + return + } + + for _, name := range t.runNames { + if !t.isRegisteredTestName(name) { + fatalf("unknown test %q", name) + } + } + + // On a few builders, make GOROOT unwritable to catch tests writing to it. + if strings.HasPrefix(os.Getenv("GO_BUILDER_NAME"), "linux-") { + if os.Getuid() == 0 { + // Don't bother making GOROOT unwritable: + // we're running as root, so permissions would have no effect. + } else { + xatexit(t.makeGOROOTUnwritable()) + } + } + + for _, dt := range t.tests { + if !t.shouldRunTest(dt.name) { + t.partial = true + continue + } + dt := dt // dt used in background after this iteration + if err := dt.fn(&dt); err != nil { + t.runPending(&dt) // in case that hasn't been done yet + t.failed = true + if t.keepGoing { + log.Printf("Failed: %v", err) + } else { + fatalf("Failed: %v", err) + } + } + } + t.runPending(nil) + timelog("end", "dist test") + + if t.failed { + fmt.Println("\nFAILED") + xexit(1) + } else if incomplete[goos+"/"+goarch] { + // The test succeeded, but consider it as failed so we don't + // forget to remove the port from the incomplete map once the + // port is complete. + fmt.Println("\nFAILED (incomplete port)") + xexit(1) + } else if t.partial { + fmt.Println("\nALL TESTS PASSED (some were excluded)") + } else { + fmt.Println("\nALL TESTS PASSED") + } +} + +func (t *tester) shouldRunTest(name string) bool { + if t.runRx != nil { + return t.runRx.MatchString(name) == t.runRxWant + } + if len(t.runNames) == 0 { + return true + } + for _, runName := range t.runNames { + if runName == name { + return true + } + } + return false +} + +// short returns a -short flag value to use with 'go test' +// or a test binary for tests intended to run in short mode. +// It returns "true", unless the environment variable +// GO_TEST_SHORT is set to a non-empty, false-ish string. +// +// This environment variable is meant to be an internal +// detail between the Go build system and cmd/dist for +// the purpose of longtest builders, and is not intended +// for use by users. See golang.org/issue/12508. +func short() string { + if v := os.Getenv("GO_TEST_SHORT"); v != "" { + short, err := strconv.ParseBool(v) + if err != nil { + fatalf("invalid GO_TEST_SHORT %q: %v", v, err) + } + if !short { + return "false" + } + } + return "true" +} + +// goTest returns the beginning of the go test command line. +// Callers should use goTest and then pass flags overriding these +// defaults as later arguments in the command line. +func (t *tester) goTest() []string { + return []string{ + "go", "test", "-short=" + short(), "-count=1", t.tags(), t.runFlag(""), + } +} + +func (t *tester) tags() string { + if t.iOS() { + return "-tags=lldb" + } + return "-tags=" +} + +// timeoutDuration converts the provided number of seconds into a +// time.Duration, scaled by the t.timeoutScale factor. +func (t *tester) timeoutDuration(sec int) time.Duration { + return time.Duration(sec) * time.Second * time.Duration(t.timeoutScale) +} + +// timeout returns the "-timeout=" string argument to "go test" given +// the number of seconds of timeout. It scales it by the +// t.timeoutScale factor. +func (t *tester) timeout(sec int) string { + return "-timeout=" + t.timeoutDuration(sec).String() +} + +// ranGoTest and stdMatches are state closed over by the stdlib +// testing func in registerStdTest below. The tests are run +// sequentially, so there's no need for locks. +// +// ranGoBench and benchMatches are the same, but are only used +// in -race mode. +var ( + ranGoTest bool + stdMatches []string + + ranGoBench bool + benchMatches []string +) + +func (t *tester) registerStdTest(pkg string, useG3 bool) { + heading := "Testing packages." + testPrefix := "go_test:" + gcflags := gogcflags + if useG3 { + heading = "Testing packages with -G=3." + testPrefix = "go_test_g3:" + gcflags += " -G=3" + } + + testName := testPrefix + pkg + if t.runRx == nil || t.runRx.MatchString(testName) == t.runRxWant { + stdMatches = append(stdMatches, pkg) + } + + t.tests = append(t.tests, distTest{ + name: testName, + heading: heading, + fn: func(dt *distTest) error { + if ranGoTest { + return nil + } + t.runPending(dt) + timelog("start", dt.name) + defer timelog("end", dt.name) + ranGoTest = true + + timeoutSec := 180 + for _, pkg := range stdMatches { + if pkg == "cmd/go" { + timeoutSec *= 3 + break + } + } + // Special case for our slow cross-compiled + // qemu builders: + if t.shouldUsePrecompiledStdTest() { + return t.runPrecompiledStdTest(t.timeoutDuration(timeoutSec)) + } + args := []string{ + "test", + "-short=" + short(), + t.tags(), + t.timeout(timeoutSec), + "-gcflags=all=" + gcflags, + } + if t.race { + args = append(args, "-race") + } + if t.compileOnly { + args = append(args, "-run=^$") + } + args = append(args, stdMatches...) + cmd := exec.Command("go", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() + }, + }) +} + +func (t *tester) registerRaceBenchTest(pkg string) { + testName := "go_test_bench:" + pkg + if t.runRx == nil || t.runRx.MatchString(testName) == t.runRxWant { + benchMatches = append(benchMatches, pkg) + } + t.tests = append(t.tests, distTest{ + name: testName, + heading: "Running benchmarks briefly.", + fn: func(dt *distTest) error { + if ranGoBench { + return nil + } + t.runPending(dt) + timelog("start", dt.name) + defer timelog("end", dt.name) + ranGoBench = true + args := []string{ + "test", + "-short=" + short(), + "-race", + t.timeout(1200), // longer timeout for race with benchmarks + "-run=^$", // nothing. only benchmarks. + "-benchtime=.1s", + "-cpu=4", + } + if !t.compileOnly { + args = append(args, "-bench=.*") + } + args = append(args, benchMatches...) + cmd := exec.Command("go", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() + }, + }) +} + +// stdOutErrAreTerminals is defined in test_linux.go, to report +// whether stdout & stderr are terminals. +var stdOutErrAreTerminals func() bool + +func (t *tester) registerTests() { + // Fast path to avoid the ~1 second of `go list std cmd` when + // the caller lists specific tests to run. (as the continuous + // build coordinator does). + if len(t.runNames) > 0 { + for _, name := range t.runNames { + if strings.HasPrefix(name, "go_test:") { + t.registerStdTest(strings.TrimPrefix(name, "go_test:"), false) + } + if strings.HasPrefix(name, "go_test_g3:") { + t.registerStdTest(strings.TrimPrefix(name, "go_test_g3:"), true) + } + if strings.HasPrefix(name, "go_test_bench:") { + t.registerRaceBenchTest(strings.TrimPrefix(name, "go_test_bench:")) + } + } + } else { + // Use a format string to only list packages and commands that have tests. + const format = "{{if (or .TestGoFiles .XTestGoFiles)}}{{.ImportPath}}{{end}}" + cmd := exec.Command("go", "list", "-f", format) + if t.race { + cmd.Args = append(cmd.Args, "-tags=race") + } + cmd.Args = append(cmd.Args, "std") + if t.shouldTestCmd() { + cmd.Args = append(cmd.Args, "cmd") + } + cmd.Stderr = new(bytes.Buffer) + all, err := cmd.Output() + if err != nil { + fatalf("Error running go list std cmd: %v:\n%s", err, cmd.Stderr) + } + pkgs := strings.Fields(string(all)) + if false { + // Disable -G=3 option for standard tests for now, since + // they are flaky on the builder. + for _, pkg := range pkgs { + t.registerStdTest(pkg, true /* -G=3 flag */) + } + } + for _, pkg := range pkgs { + t.registerStdTest(pkg, false) + } + if t.race { + for _, pkg := range pkgs { + if t.packageHasBenchmarks(pkg) { + t.registerRaceBenchTest(pkg) + } + } + } + } + + // Test the os/user package in the pure-Go mode too. + if !t.compileOnly { + t.tests = append(t.tests, distTest{ + name: "osusergo", + heading: "os/user with tag osusergo", + fn: func(dt *distTest) error { + t.addCmd(dt, "src", t.goTest(), t.timeout(300), "-tags=osusergo", "os/user") + return nil + }, + }) + } + + // Test ios/amd64 for the iOS simulator. + if goos == "darwin" && goarch == "amd64" && t.cgoEnabled { + t.tests = append(t.tests, distTest{ + name: "amd64ios", + heading: "GOOS=ios on darwin/amd64", + fn: func(dt *distTest) error { + cmd := t.addCmd(dt, "src", t.goTest(), t.timeout(300), "-run=SystemRoots", "crypto/x509") + setEnv(cmd, "GOOS", "ios") + setEnv(cmd, "CGO_ENABLED", "1") + return nil + }, + }) + } + + if t.race { + return + } + + // Runtime CPU tests. + if !t.compileOnly && goos != "js" { // js can't handle -cpu != 1 + testName := "runtime:cpu124" + t.tests = append(t.tests, distTest{ + name: testName, + heading: "GOMAXPROCS=2 runtime -cpu=1,2,4 -quick", + fn: func(dt *distTest) error { + cmd := t.addCmd(dt, "src", t.goTest(), t.timeout(300), "runtime", "-cpu=1,2,4", "-quick") + // We set GOMAXPROCS=2 in addition to -cpu=1,2,4 in order to test runtime bootstrap code, + // creation of first goroutines and first garbage collections in the parallel setting. + setEnv(cmd, "GOMAXPROCS", "2") + return nil + }, + }) + } + + // This test needs its stdout/stderr to be terminals, so we don't run it from cmd/go's tests. + // See issue 18153. + if goos == "linux" { + t.tests = append(t.tests, distTest{ + name: "cmd_go_test_terminal", + heading: "cmd/go terminal test", + fn: func(dt *distTest) error { + t.runPending(dt) + timelog("start", dt.name) + defer timelog("end", dt.name) + if !stdOutErrAreTerminals() { + fmt.Println("skipping terminal test; stdout/stderr not terminals") + return nil + } + cmd := exec.Command("go", "test") + setDir(cmd, filepath.Join(os.Getenv("GOROOT"), "src/cmd/go/testdata/testterminal18153")) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() + }, + }) + } + + // On the builders only, test that a moved GOROOT still works. + // Fails on iOS because CC_FOR_TARGET refers to clangwrap.sh + // in the unmoved GOROOT. + // Fails on Android and js/wasm with an exec format error. + // Fails on plan9 with "cannot find GOROOT" (issue #21016). + if os.Getenv("GO_BUILDER_NAME") != "" && goos != "android" && !t.iOS() && goos != "plan9" && goos != "js" { + t.tests = append(t.tests, distTest{ + name: "moved_goroot", + heading: "moved GOROOT", + fn: func(dt *distTest) error { + t.runPending(dt) + timelog("start", dt.name) + defer timelog("end", dt.name) + moved := goroot + "-moved" + if err := os.Rename(goroot, moved); err != nil { + if goos == "windows" { + // Fails on Windows (with "Access is denied") if a process + // or binary is in this directory. For instance, using all.bat + // when run from c:\workdir\go\src fails here + // if GO_BUILDER_NAME is set. Our builders invoke tests + // a different way which happens to work when sharding + // tests, but we should be tolerant of the non-sharded + // all.bat case. + log.Printf("skipping test on Windows") + return nil + } + return err + } + + // Run `go test fmt` in the moved GOROOT, without explicitly setting + // GOROOT in the environment. The 'go' command should find itself. + cmd := exec.Command(filepath.Join(moved, "bin", "go"), "test", "fmt") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + unsetEnv(cmd, "GOROOT") + unsetEnv(cmd, "GOCACHE") // TODO(bcmills): ...why‽ + err := cmd.Run() + + if rerr := os.Rename(moved, goroot); rerr != nil { + fatalf("failed to restore GOROOT: %v", rerr) + } + return err + }, + }) + } + + // Test that internal linking of standard packages does not + // require libgcc. This ensures that we can install a Go + // release on a system that does not have a C compiler + // installed and still build Go programs (that don't use cgo). + for _, pkg := range cgoPackages { + if !t.internalLink() { + break + } + + // ARM libgcc may be Thumb, which internal linking does not support. + if goarch == "arm" { + break + } + + pkg := pkg + var run string + if pkg == "net" { + run = "TestTCPStress" + } + t.tests = append(t.tests, distTest{ + name: "nolibgcc:" + pkg, + heading: "Testing without libgcc.", + fn: func(dt *distTest) error { + // What matters is that the tests build and start up. + // Skip expensive tests, especially x509 TestSystemRoots. + t.addCmd(dt, "src", t.goTest(), "-ldflags=-linkmode=internal -libgcc=none", "-run=^Test[^CS]", pkg, t.runFlag(run)) + return nil + }, + }) + } + + // Test internal linking of PIE binaries where it is supported. + if t.internalLinkPIE() { + t.tests = append(t.tests, distTest{ + name: "pie_internal", + heading: "internal linking of -buildmode=pie", + fn: func(dt *distTest) error { + t.addCmd(dt, "src", t.goTest(), "reflect", "-buildmode=pie", "-ldflags=-linkmode=internal", t.timeout(60)) + return nil + }, + }) + // Also test a cgo package. + if t.cgoEnabled && t.internalLink() { + t.tests = append(t.tests, distTest{ + name: "pie_internal_cgo", + heading: "internal linking of -buildmode=pie", + fn: func(dt *distTest) error { + t.addCmd(dt, "src", t.goTest(), "os/user", "-buildmode=pie", "-ldflags=-linkmode=internal", t.timeout(60)) + return nil + }, + }) + } + } + + // sync tests + if goos != "js" { // js doesn't support -cpu=10 + t.tests = append(t.tests, distTest{ + name: "sync_cpu", + heading: "sync -cpu=10", + fn: func(dt *distTest) error { + t.addCmd(dt, "src", t.goTest(), "sync", t.timeout(120), "-cpu=10", t.runFlag("")) + return nil + }, + }) + } + + if t.raceDetectorSupported() { + t.tests = append(t.tests, distTest{ + name: "race", + heading: "Testing race detector", + fn: t.raceTest, + }) + } + + if t.cgoEnabled && !t.iOS() { + // Disabled on iOS. golang.org/issue/15919 + t.registerHostTest("cgo_stdio", "../misc/cgo/stdio", "misc/cgo/stdio", ".") + t.registerHostTest("cgo_life", "../misc/cgo/life", "misc/cgo/life", ".") + fortran := os.Getenv("FC") + if fortran == "" { + fortran, _ = exec.LookPath("gfortran") + } + if t.hasBash() && goos != "android" && fortran != "" { + t.tests = append(t.tests, distTest{ + name: "cgo_fortran", + heading: "../misc/cgo/fortran", + fn: func(dt *distTest) error { + t.addCmd(dt, "misc/cgo/fortran", "./test.bash", fortran) + return nil + }, + }) + } + if t.hasSwig() && goos != "android" { + t.tests = append(t.tests, distTest{ + name: "swig_stdio", + heading: "../misc/swig/stdio", + fn: func(dt *distTest) error { + t.addCmd(dt, "misc/swig/stdio", t.goTest()) + return nil + }, + }) + if t.hasCxx() { + t.tests = append(t.tests, + distTest{ + name: "swig_callback", + heading: "../misc/swig/callback", + fn: func(dt *distTest) error { + t.addCmd(dt, "misc/swig/callback", t.goTest()) + return nil + }, + }, + distTest{ + name: "swig_callback_lto", + heading: "../misc/swig/callback", + fn: func(dt *distTest) error { + cmd := t.addCmd(dt, "misc/swig/callback", t.goTest()) + setEnv(cmd, "CGO_CFLAGS", "-flto -Wno-lto-type-mismatch -Wno-unknown-warning-option") + setEnv(cmd, "CGO_CXXFLAGS", "-flto -Wno-lto-type-mismatch -Wno-unknown-warning-option") + setEnv(cmd, "CGO_LDFLAGS", "-flto -Wno-lto-type-mismatch -Wno-unknown-warning-option") + return nil + }, + }, + ) + } + } + } + if t.cgoEnabled { + t.tests = append(t.tests, distTest{ + name: "cgo_test", + heading: "../misc/cgo/test", + fn: t.cgoTest, + }) + } + + // Don't run these tests with $GO_GCFLAGS because most of them + // assume that they can run "go install" with no -gcflags and not + // recompile the entire standard library. If make.bash ran with + // special -gcflags, that's not true. + if t.cgoEnabled && gogcflags == "" { + t.registerHostTest("testgodefs", "../misc/cgo/testgodefs", "misc/cgo/testgodefs", ".") + + t.registerTest("testso", "../misc/cgo/testso", t.goTest(), t.timeout(600), ".") + t.registerTest("testsovar", "../misc/cgo/testsovar", t.goTest(), t.timeout(600), ".") + if t.supportedBuildmode("c-archive") { + t.registerHostTest("testcarchive", "../misc/cgo/testcarchive", "misc/cgo/testcarchive", ".") + } + if t.supportedBuildmode("c-shared") { + t.registerHostTest("testcshared", "../misc/cgo/testcshared", "misc/cgo/testcshared", ".") + } + if t.supportedBuildmode("shared") { + t.registerTest("testshared", "../misc/cgo/testshared", t.goTest(), t.timeout(600), ".") + } + if t.supportedBuildmode("plugin") { + t.registerTest("testplugin", "../misc/cgo/testplugin", t.goTest(), t.timeout(600), ".") + } + if gohostos == "linux" && goarch == "amd64" { + t.registerTest("testasan", "../misc/cgo/testasan", "go", "run", ".") + } + if goos == "linux" && goarch != "ppc64le" { + // because syscall.SysProcAttr struct used in misc/cgo/testsanitizers is only built on linux. + // Some inconsistent failures happen on ppc64le so disable for now. + t.registerHostTest("testsanitizers", "../misc/cgo/testsanitizers", "misc/cgo/testsanitizers", ".") + } + if t.hasBash() && goos != "android" && !t.iOS() && gohostos != "windows" { + t.registerHostTest("cgo_errors", "../misc/cgo/errors", "misc/cgo/errors", ".") + } + if gohostos == "linux" && t.extLink() { + t.registerTest("testsigfwd", "../misc/cgo/testsigfwd", "go", "run", ".") + } + } + + if goos != "android" && !t.iOS() { + // There are no tests in this directory, only benchmarks. + // Check that the test binary builds but don't bother running it. + // (It has init-time work to set up for the benchmarks that is not worth doing unnecessarily.) + t.registerTest("bench_go1", "../test/bench/go1", t.goTest(), "-c", "-o="+os.DevNull) + } + if goos != "android" && !t.iOS() { + // Only start multiple test dir shards on builders, + // where they get distributed to multiple machines. + // See issues 20141 and 31834. + nShards := 1 + if os.Getenv("GO_BUILDER_NAME") != "" { + nShards = 10 + } + if n, err := strconv.Atoi(os.Getenv("GO_TEST_SHARDS")); err == nil { + nShards = n + } + for shard := 0; shard < nShards; shard++ { + shard := shard + t.tests = append(t.tests, distTest{ + name: fmt.Sprintf("test:%d_%d", shard, nShards), + heading: "../test", + fn: func(dt *distTest) error { return t.testDirTest(dt, shard, nShards) }, + }) + } + } + // Only run the API check on fast development platforms. Android, iOS, and JS + // are always cross-compiled, and the filesystems on our only plan9 builders + // are too slow to complete in a reasonable timeframe. Every platform checks + // the API on every GOOS/GOARCH/CGO_ENABLED combination anyway, so we really + // only need to run this check once anywhere to get adequate coverage. + if goos != "android" && !t.iOS() && goos != "js" && goos != "plan9" { + t.tests = append(t.tests, distTest{ + name: "api", + heading: "API check", + fn: func(dt *distTest) error { + if t.compileOnly { + t.addCmd(dt, "src", "go", "build", "-o", os.DevNull, filepath.Join(goroot, "src/cmd/api/run.go")) + return nil + } + t.addCmd(dt, "src", "go", "run", filepath.Join(goroot, "src/cmd/api/run.go")) + return nil + }, + }) + } + + // Ensure that the toolchain can bootstrap itself. + // This test adds another ~45s to all.bash if run sequentially, so run it only on the builders. + if os.Getenv("GO_BUILDER_NAME") != "" && goos != "android" && !t.iOS() { + t.registerHostTest("reboot", "../misc/reboot", "misc/reboot", ".") + } +} + +// isRegisteredTestName reports whether a test named testName has already +// been registered. +func (t *tester) isRegisteredTestName(testName string) bool { + for _, tt := range t.tests { + if tt.name == testName { + return true + } + } + return false +} + +func (t *tester) registerTest1(seq bool, name, dirBanner string, cmdline ...interface{}) { + bin, args := flattenCmdline(cmdline) + if bin == "time" && !t.haveTime { + bin, args = args[0], args[1:] + } + if t.isRegisteredTestName(name) { + panic("duplicate registered test name " + name) + } + t.tests = append(t.tests, distTest{ + name: name, + heading: dirBanner, + fn: func(dt *distTest) error { + if seq { + t.runPending(dt) + timelog("start", name) + defer timelog("end", name) + return t.dirCmd(filepath.Join(goroot, "src", dirBanner), bin, args).Run() + } + t.addCmd(dt, filepath.Join(goroot, "src", dirBanner), bin, args) + return nil + }, + }) +} + +func (t *tester) registerTest(name, dirBanner string, cmdline ...interface{}) { + t.registerTest1(false, name, dirBanner, cmdline...) +} + +func (t *tester) registerSeqTest(name, dirBanner string, cmdline ...interface{}) { + t.registerTest1(true, name, dirBanner, cmdline...) +} + +func (t *tester) bgDirCmd(dir, bin string, args ...string) *exec.Cmd { + cmd := exec.Command(bin, args...) + if filepath.IsAbs(dir) { + setDir(cmd, dir) + } else { + setDir(cmd, filepath.Join(goroot, dir)) + } + return cmd +} + +func (t *tester) dirCmd(dir string, cmdline ...interface{}) *exec.Cmd { + bin, args := flattenCmdline(cmdline) + cmd := t.bgDirCmd(dir, bin, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if vflag > 1 { + errprintf("%s\n", strings.Join(cmd.Args, " ")) + } + return cmd +} + +// flattenCmdline flattens a mixture of string and []string as single list +// and then interprets it as a command line: first element is binary, then args. +func flattenCmdline(cmdline []interface{}) (bin string, args []string) { + var list []string + for _, x := range cmdline { + switch x := x.(type) { + case string: + list = append(list, x) + case []string: + list = append(list, x...) + default: + panic("invalid addCmd argument type: " + reflect.TypeOf(x).String()) + } + } + + // The go command is too picky about duplicated flags. + // Drop all but the last of the allowed duplicated flags. + drop := make([]bool, len(list)) + have := map[string]int{} + for i := 1; i < len(list); i++ { + j := strings.Index(list[i], "=") + if j < 0 { + continue + } + flag := list[i][:j] + switch flag { + case "-run", "-tags": + if have[flag] != 0 { + drop[have[flag]] = true + } + have[flag] = i + } + } + out := list[:0] + for i, x := range list { + if !drop[i] { + out = append(out, x) + } + } + list = out + + return list[0], list[1:] +} + +func (t *tester) addCmd(dt *distTest, dir string, cmdline ...interface{}) *exec.Cmd { + bin, args := flattenCmdline(cmdline) + w := &work{ + dt: dt, + cmd: t.bgDirCmd(dir, bin, args...), + } + t.worklist = append(t.worklist, w) + return w.cmd +} + +func (t *tester) iOS() bool { + return goos == "ios" +} + +func (t *tester) out(v string) { + if t.banner == "" { + return + } + fmt.Println("\n" + t.banner + v) +} + +func (t *tester) extLink() bool { + pair := gohostos + "-" + goarch + switch pair { + case "aix-ppc64", + "android-arm", "android-arm64", + "darwin-amd64", "darwin-arm64", + "dragonfly-amd64", + "freebsd-386", "freebsd-amd64", "freebsd-arm", + "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-mips64", "linux-mips64le", "linux-mips", "linux-mipsle", "linux-riscv64", "linux-s390x", + "netbsd-386", "netbsd-amd64", + "openbsd-386", "openbsd-amd64", + "windows-386", "windows-amd64": + return true + } + return false +} + +func (t *tester) internalLink() bool { + if gohostos == "dragonfly" { + // linkmode=internal fails on dragonfly since errno is a TLS relocation. + return false + } + if goos == "android" { + return false + } + if goos == "ios" { + return false + } + if goos == "windows" && goarch == "arm64" { + return false + } + // Internally linking cgo is incomplete on some architectures. + // https://golang.org/issue/10373 + // https://golang.org/issue/14449 + if goarch == "mips64" || goarch == "mips64le" || goarch == "mips" || goarch == "mipsle" || goarch == "riscv64" { + return false + } + if goos == "aix" { + // linkmode=internal isn't supported. + return false + } + return true +} + +func (t *tester) internalLinkPIE() bool { + switch goos + "-" + goarch { + case "darwin-amd64", "darwin-arm64", + "linux-amd64", "linux-arm64", "linux-ppc64le", + "android-arm64", + "windows-amd64", "windows-386", "windows-arm": + return true + } + return false +} + +func (t *tester) supportedBuildmode(mode string) bool { + pair := goos + "-" + goarch + switch mode { + case "c-archive": + if !t.extLink() { + return false + } + switch pair { + case "aix-ppc64", + "darwin-amd64", "darwin-arm64", "ios-arm64", + "linux-amd64", "linux-386", "linux-ppc64le", "linux-riscv64", "linux-s390x", + "freebsd-amd64", + "windows-amd64", "windows-386": + return true + } + return false + case "c-shared": + switch pair { + case "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-riscv64", "linux-s390x", + "darwin-amd64", "darwin-arm64", + "freebsd-amd64", + "android-arm", "android-arm64", "android-386", + "windows-amd64", "windows-386", "windows-arm64": + return true + } + return false + case "shared": + switch pair { + case "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-s390x": + return true + } + return false + case "plugin": + switch pair { + case "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-s390x", "linux-ppc64le": + return true + case "darwin-amd64", "darwin-arm64": + return true + case "freebsd-amd64": + return true + } + return false + case "pie": + switch pair { + case "aix/ppc64", + "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-riscv64", "linux-s390x", + "android-amd64", "android-arm", "android-arm64", "android-386": + return true + case "darwin-amd64", "darwin-arm64": + return true + case "windows-amd64", "windows-386", "windows-arm": + return true + } + return false + + default: + fatalf("internal error: unknown buildmode %s", mode) + return false + } +} + +func (t *tester) registerHostTest(name, heading, dir, pkg string) { + t.tests = append(t.tests, distTest{ + name: name, + heading: heading, + fn: func(dt *distTest) error { + t.runPending(dt) + timelog("start", name) + defer timelog("end", name) + return t.runHostTest(dir, pkg) + }, + }) +} + +func (t *tester) runHostTest(dir, pkg string) error { + out, err := exec.Command("go", "env", "GOEXE", "GOTMPDIR").Output() + if err != nil { + return err + } + + parts := strings.Split(string(out), "\n") + if len(parts) < 2 { + return fmt.Errorf("'go env GOEXE GOTMPDIR' output contains <2 lines") + } + GOEXE := strings.TrimSpace(parts[0]) + GOTMPDIR := strings.TrimSpace(parts[1]) + + f, err := ioutil.TempFile(GOTMPDIR, "test.test-*"+GOEXE) + if err != nil { + return err + } + f.Close() + defer os.Remove(f.Name()) + + cmd := t.dirCmd(dir, t.goTest(), "-c", "-o", f.Name(), pkg) + setEnv(cmd, "GOARCH", gohostarch) + setEnv(cmd, "GOOS", gohostos) + if err := cmd.Run(); err != nil { + return err + } + return t.dirCmd(dir, f.Name(), "-test.short="+short()).Run() +} + +func (t *tester) cgoTest(dt *distTest) error { + cmd := t.addCmd(dt, "misc/cgo/test", t.goTest()) + setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=auto") + + // Skip internal linking cases on arm64 to support GCC-9.4 and above. + // See issue #39466. + skipInternalLink := goarch == "arm64" && goos != "darwin" + + if t.internalLink() && !skipInternalLink { + cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), "-tags=internal") + setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=internal") + } + + pair := gohostos + "-" + goarch + switch pair { + case "darwin-amd64", "darwin-arm64", + "windows-386", "windows-amd64": + // test linkmode=external, but __thread not supported, so skip testtls. + if !t.extLink() { + break + } + cmd := t.addCmd(dt, "misc/cgo/test", t.goTest()) + setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=external") + + t.addCmd(dt, "misc/cgo/test", t.goTest(), "-ldflags", "-linkmode=external -s") + + if t.supportedBuildmode("pie") { + t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie") + if t.internalLink() && t.internalLinkPIE() { + t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie", "-ldflags=-linkmode=internal", "-tags=internal,internal_pie") + } + } + + case "aix-ppc64", + "android-arm", "android-arm64", + "dragonfly-amd64", + "freebsd-386", "freebsd-amd64", "freebsd-arm", + "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-riscv64", "linux-s390x", + "netbsd-386", "netbsd-amd64", + "openbsd-386", "openbsd-amd64", "openbsd-arm", "openbsd-arm64", "openbsd-mips64": + + cmd := t.addCmd(dt, "misc/cgo/test", t.goTest()) + setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=external") + // cgo should be able to cope with both -g arguments and colored + // diagnostics. + setEnv(cmd, "CGO_CFLAGS", "-g0 -fdiagnostics-color") + + t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-ldflags", "-linkmode=auto") + t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-ldflags", "-linkmode=external") + + switch pair { + case "aix-ppc64", "netbsd-386", "netbsd-amd64": + // no static linking + case "freebsd-arm": + // -fPIC compiled tls code will use __tls_get_addr instead + // of __aeabi_read_tp, however, on FreeBSD/ARM, __tls_get_addr + // is implemented in rtld-elf, so -fPIC isn't compatible with + // static linking on FreeBSD/ARM with clang. (cgo depends on + // -fPIC fundamentally.) + default: + cmd := t.dirCmd("misc/cgo/test", + compilerEnvLookup(defaultcc, goos, goarch), "-xc", "-o", "/dev/null", "-static", "-") + cmd.Stdin = strings.NewReader("int main() {}") + if err := cmd.Run(); err != nil { + fmt.Println("No support for static linking found (lacks libc.a?), skip cgo static linking test.") + } else { + if goos != "android" { + t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-ldflags", `-linkmode=external -extldflags "-static -pthread"`) + } + t.addCmd(dt, "misc/cgo/nocgo", t.goTest()) + t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), "-ldflags", `-linkmode=external`) + if goos != "android" { + t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), "-ldflags", `-linkmode=external -extldflags "-static -pthread"`) + t.addCmd(dt, "misc/cgo/test", t.goTest(), "-tags=static", "-ldflags", `-linkmode=external -extldflags "-static -pthread"`) + // -static in CGO_LDFLAGS triggers a different code path + // than -static in -extldflags, so test both. + // See issue #16651. + cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), "-tags=static") + setEnv(cmd, "CGO_LDFLAGS", "-static -pthread") + } + } + + if t.supportedBuildmode("pie") { + t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie") + if t.internalLink() && t.internalLinkPIE() && !skipInternalLink { + t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie", "-ldflags=-linkmode=internal", "-tags=internal,internal_pie") + } + t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-buildmode=pie") + t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), "-buildmode=pie") + } + } + } + + return nil +} + +// run pending test commands, in parallel, emitting headers as appropriate. +// When finished, emit header for nextTest, which is going to run after the +// pending commands are done (and runPending returns). +// A test should call runPending if it wants to make sure that it is not +// running in parallel with earlier tests, or if it has some other reason +// for needing the earlier tests to be done. +func (t *tester) runPending(nextTest *distTest) { + checkNotStale("go", "std") + worklist := t.worklist + t.worklist = nil + for _, w := range worklist { + w.start = make(chan bool) + w.end = make(chan bool) + go func(w *work) { + if !<-w.start { + timelog("skip", w.dt.name) + w.out = []byte(fmt.Sprintf("skipped due to earlier error\n")) + } else { + timelog("start", w.dt.name) + w.out, w.err = w.cmd.CombinedOutput() + if w.err != nil { + if isUnsupportedVMASize(w) { + timelog("skip", w.dt.name) + w.out = []byte(fmt.Sprintf("skipped due to unsupported VMA\n")) + w.err = nil + } + } + } + timelog("end", w.dt.name) + w.end <- true + }(w) + } + + started := 0 + ended := 0 + var last *distTest + for ended < len(worklist) { + for started < len(worklist) && started-ended < maxbg { + w := worklist[started] + started++ + w.start <- !t.failed || t.keepGoing + } + w := worklist[ended] + dt := w.dt + if dt.heading != "" && t.lastHeading != dt.heading { + t.lastHeading = dt.heading + t.out(dt.heading) + } + if dt != last { + // Assumes all the entries for a single dt are in one worklist. + last = w.dt + if vflag > 0 { + fmt.Printf("# go tool dist test -run=^%s$\n", dt.name) + } + } + if vflag > 1 { + errprintf("%s\n", strings.Join(w.cmd.Args, " ")) + } + ended++ + <-w.end + os.Stdout.Write(w.out) + if w.err != nil { + log.Printf("Failed: %v", w.err) + t.failed = true + } + checkNotStale("go", "std") + } + if t.failed && !t.keepGoing { + fatalf("FAILED") + } + + if dt := nextTest; dt != nil { + if dt.heading != "" && t.lastHeading != dt.heading { + t.lastHeading = dt.heading + t.out(dt.heading) + } + if vflag > 0 { + fmt.Printf("# go tool dist test -run=^%s$\n", dt.name) + } + } +} + +func (t *tester) hasBash() bool { + switch gohostos { + case "windows", "plan9": + return false + } + return true +} + +func (t *tester) hasCxx() bool { + cxx, _ := exec.LookPath(compilerEnvLookup(defaultcxx, goos, goarch)) + return cxx != "" +} + +func (t *tester) hasSwig() bool { + swig, err := exec.LookPath("swig") + if err != nil { + return false + } + + // Check that swig was installed with Go support by checking + // that a go directory exists inside the swiglib directory. + // See https://golang.org/issue/23469. + output, err := exec.Command(swig, "-go", "-swiglib").Output() + if err != nil { + return false + } + swigDir := strings.TrimSpace(string(output)) + + _, err = os.Stat(filepath.Join(swigDir, "go")) + if err != nil { + return false + } + + // Check that swig has a new enough version. + // See https://golang.org/issue/22858. + out, err := exec.Command(swig, "-version").CombinedOutput() + if err != nil { + return false + } + + re := regexp.MustCompile(`[vV]ersion +([\d]+)([.][\d]+)?([.][\d]+)?`) + matches := re.FindSubmatch(out) + if matches == nil { + // Can't find version number; hope for the best. + return true + } + + major, err := strconv.Atoi(string(matches[1])) + if err != nil { + // Can't find version number; hope for the best. + return true + } + if major < 3 { + return false + } + if major > 3 { + // 4.0 or later + return true + } + + // We have SWIG version 3.x. + if len(matches[2]) > 0 { + minor, err := strconv.Atoi(string(matches[2][1:])) + if err != nil { + return true + } + if minor > 0 { + // 3.1 or later + return true + } + } + + // We have SWIG version 3.0.x. + if len(matches[3]) > 0 { + patch, err := strconv.Atoi(string(matches[3][1:])) + if err != nil { + return true + } + if patch < 6 { + // Before 3.0.6. + return false + } + } + + return true +} + +func (t *tester) raceDetectorSupported() bool { + if gohostos != goos { + return false + } + if !t.cgoEnabled { + return false + } + if !raceDetectorSupported(goos, goarch) { + return false + } + // The race detector doesn't work on Alpine Linux: + // golang.org/issue/14481 + if isAlpineLinux() { + return false + } + // NetBSD support is unfinished. + // golang.org/issue/26403 + if goos == "netbsd" { + return false + } + return true +} + +func isAlpineLinux() bool { + if runtime.GOOS != "linux" { + return false + } + fi, err := os.Lstat("/etc/alpine-release") + return err == nil && fi.Mode().IsRegular() +} + +func (t *tester) runFlag(rx string) string { + if t.compileOnly { + return "-run=^$" + } + return "-run=" + rx +} + +func (t *tester) raceTest(dt *distTest) error { + t.addCmd(dt, "src", t.goTest(), "-race", t.runFlag("Output"), "runtime/race") + t.addCmd(dt, "src", t.goTest(), "-race", t.runFlag("TestParse|TestEcho|TestStdinCloseRace|TestClosedPipeRace|TestTypeRace|TestFdRace|TestFdReadRace|TestFileCloseRace"), "flag", "net", "os", "os/exec", "encoding/gob") + // We don't want the following line, because it + // slows down all.bash (by 10 seconds on my laptop). + // The race builder should catch any error here, but doesn't. + // TODO(iant): Figure out how to catch this. + // t.addCmd(dt, "src", t.goTest(), "-race", "-run=TestParallelTest", "cmd/go") + if t.cgoEnabled { + // Building misc/cgo/test takes a long time. + // There are already cgo-enabled packages being tested with the race detector. + // We shouldn't need to redo all of misc/cgo/test too. + // The race buildler will take care of this. + // cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), "-race") + // setEnv(cmd, "GOTRACEBACK", "2") + } + if t.extLink() { + // Test with external linking; see issue 9133. + t.addCmd(dt, "src", t.goTest(), "-race", "-ldflags=-linkmode=external", t.runFlag("TestParse|TestEcho|TestStdinCloseRace"), "flag", "os/exec") + } + return nil +} + +var runtest struct { + sync.Once + exe string + err error +} + +func (t *tester) testDirTest(dt *distTest, shard, shards int) error { + runtest.Do(func() { + f, err := ioutil.TempFile("", "runtest-*.exe") // named exe for Windows, but harmless elsewhere + if err != nil { + runtest.err = err + return + } + f.Close() + + runtest.exe = f.Name() + xatexit(func() { + os.Remove(runtest.exe) + }) + + cmd := t.dirCmd("test", "go", "build", "-o", runtest.exe, "run.go") + setEnv(cmd, "GOOS", gohostos) + setEnv(cmd, "GOARCH", gohostarch) + runtest.err = cmd.Run() + }) + if runtest.err != nil { + return runtest.err + } + if t.compileOnly { + return nil + } + t.addCmd(dt, "test", runtest.exe, + fmt.Sprintf("--shard=%d", shard), + fmt.Sprintf("--shards=%d", shards), + ) + return nil +} + +// cgoPackages is the standard packages that use cgo. +var cgoPackages = []string{ + "net", + "os/user", +} + +var funcBenchmark = []byte("\nfunc Benchmark") + +// packageHasBenchmarks reports whether pkg has benchmarks. +// On any error, it conservatively returns true. +// +// This exists just to eliminate work on the builders, since compiling +// a test in race mode just to discover it has no benchmarks costs a +// second or two per package, and this function returns false for +// about 100 packages. +func (t *tester) packageHasBenchmarks(pkg string) bool { + pkgDir := filepath.Join(goroot, "src", pkg) + d, err := os.Open(pkgDir) + if err != nil { + return true // conservatively + } + defer d.Close() + names, err := d.Readdirnames(-1) + if err != nil { + return true // conservatively + } + for _, name := range names { + if !strings.HasSuffix(name, "_test.go") { + continue + } + slurp, err := ioutil.ReadFile(filepath.Join(pkgDir, name)) + if err != nil { + return true // conservatively + } + if bytes.Contains(slurp, funcBenchmark) { + return true + } + } + return false +} + +// makeGOROOTUnwritable makes all $GOROOT files & directories non-writable to +// check that no tests accidentally write to $GOROOT. +func (t *tester) makeGOROOTUnwritable() (undo func()) { + dir := os.Getenv("GOROOT") + if dir == "" { + panic("GOROOT not set") + } + + type pathMode struct { + path string + mode os.FileMode + } + var dirs []pathMode // in lexical order + + undo = func() { + for i := range dirs { + os.Chmod(dirs[i].path, dirs[i].mode) // best effort + } + } + + gocache := os.Getenv("GOCACHE") + if gocache == "" { + panic("GOCACHE not set") + } + gocacheSubdir, _ := filepath.Rel(dir, gocache) + + // Note: Can't use WalkDir here, because this has to compile with Go 1.4. + filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if suffix := strings.TrimPrefix(path, dir+string(filepath.Separator)); suffix != "" { + if suffix == gocacheSubdir { + // Leave GOCACHE writable: we may need to write test binaries into it. + return filepath.SkipDir + } + if suffix == ".git" { + // Leave Git metadata in whatever state it was in. It may contain a lot + // of files, and it is highly unlikely that a test will try to modify + // anything within that directory. + return filepath.SkipDir + } + } + if err == nil { + mode := info.Mode() + if mode&0222 != 0 && (mode.IsDir() || mode.IsRegular()) { + dirs = append(dirs, pathMode{path, mode}) + } + } + return nil + }) + + // Run over list backward to chmod children before parents. + for i := len(dirs) - 1; i >= 0; i-- { + err := os.Chmod(dirs[i].path, dirs[i].mode&^0222) + if err != nil { + dirs = dirs[i:] // Only undo what we did so far. + undo() + fatalf("failed to make GOROOT read-only: %v", err) + } + } + + return undo +} + +// shouldUsePrecompiledStdTest reports whether "dist test" should use +// a pre-compiled go test binary on disk rather than running "go test" +// and compiling it again. This is used by our slow qemu-based builder +// that do full processor emulation where we cross-compile the +// make.bash step as well as pre-compile each std test binary. +// +// This only reports true if dist is run with an single go_test:foo +// argument (as the build coordinator does with our slow qemu-based +// builders), we're in a builder environment ("GO_BUILDER_NAME" is set), +// and the pre-built test binary exists. +func (t *tester) shouldUsePrecompiledStdTest() bool { + bin := t.prebuiltGoPackageTestBinary() + if bin == "" { + return false + } + _, err := os.Stat(bin) + return err == nil +} + +func (t *tester) shouldTestCmd() bool { + if goos == "js" && goarch == "wasm" { + // Issues 25911, 35220 + return false + } + return true +} + +// prebuiltGoPackageTestBinary returns the path where we'd expect +// the pre-built go test binary to be on disk when dist test is run with +// a single argument. +// It returns an empty string if a pre-built binary should not be used. +func (t *tester) prebuiltGoPackageTestBinary() string { + if len(stdMatches) != 1 || t.race || t.compileOnly || os.Getenv("GO_BUILDER_NAME") == "" { + return "" + } + pkg := stdMatches[0] + return filepath.Join(os.Getenv("GOROOT"), "src", pkg, path.Base(pkg)+".test") +} + +// runPrecompiledStdTest runs the pre-compiled standard library package test binary. +// See shouldUsePrecompiledStdTest above; it must return true for this to be called. +func (t *tester) runPrecompiledStdTest(timeout time.Duration) error { + bin := t.prebuiltGoPackageTestBinary() + fmt.Fprintf(os.Stderr, "# %s: using pre-built %s...\n", stdMatches[0], bin) + cmd := exec.Command(bin, "-test.short="+short(), "-test.timeout="+timeout.String()) + setDir(cmd, filepath.Dir(bin)) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + return err + } + // And start a timer to kill the process if it doesn't kill + // itself in the prescribed timeout. + const backupKillFactor = 1.05 // add 5% + timer := time.AfterFunc(time.Duration(float64(timeout)*backupKillFactor), func() { + fmt.Fprintf(os.Stderr, "# %s: timeout running %s; killing...\n", stdMatches[0], bin) + cmd.Process.Kill() + }) + defer timer.Stop() + return cmd.Wait() +} + +// raceDetectorSupported is a copy of the function +// cmd/internal/sys.RaceDetectorSupported, which can't be used here +// because cmd/dist has to be buildable by Go 1.4. +// The race detector only supports 48-bit VMA on arm64. But we don't have +// a good solution to check VMA size(See https://golang.org/issue/29948) +// raceDetectorSupported will always return true for arm64. But race +// detector tests may abort on non 48-bit VMA configuration, the tests +// will be marked as "skipped" in this case. +func raceDetectorSupported(goos, goarch string) bool { + switch goos { + case "linux": + return goarch == "amd64" || goarch == "ppc64le" || goarch == "arm64" + case "darwin": + return goarch == "amd64" || goarch == "arm64" + case "freebsd", "netbsd", "openbsd", "windows": + return goarch == "amd64" + default: + return false + } +} + +// isUnsupportedVMASize reports whether the failure is caused by an unsupported +// VMA for the race detector (for example, running the race detector on an +// arm64 machine configured with 39-bit VMA) +func isUnsupportedVMASize(w *work) bool { + unsupportedVMA := []byte("unsupported VMA range") + return w.dt.name == "race" && bytes.Contains(w.out, unsupportedVMA) +} + +// isEnvSet reports whether the environment variable evar is +// set in the environment. +func isEnvSet(evar string) bool { + evarEq := evar + "=" + for _, e := range os.Environ() { + if strings.HasPrefix(e, evarEq) { + return true + } + } + return false +} diff --git a/src/cmd/dist/test_linux.go b/src/cmd/dist/test_linux.go new file mode 100644 index 0000000..43d28dc --- /dev/null +++ b/src/cmd/dist/test_linux.go @@ -0,0 +1,28 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux +// +build linux + +package main + +import ( + "syscall" + "unsafe" +) + +const ioctlReadTermios = syscall.TCGETS + +// isTerminal reports whether fd is a terminal. +func isTerminal(fd uintptr) bool { + var termios syscall.Termios + _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0) + return err == 0 +} + +func init() { + stdOutErrAreTerminals = func() bool { + return isTerminal(1) && isTerminal(2) + } +} diff --git a/src/cmd/dist/util.go b/src/cmd/dist/util.go new file mode 100644 index 0000000..8856f46 --- /dev/null +++ b/src/cmd/dist/util.go @@ -0,0 +1,488 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +// pathf is fmt.Sprintf for generating paths +// (on windows it turns / into \ after the printf). +func pathf(format string, args ...interface{}) string { + return filepath.Clean(fmt.Sprintf(format, args...)) +} + +// filter returns a slice containing the elements x from list for which f(x) == true. +func filter(list []string, f func(string) bool) []string { + var out []string + for _, x := range list { + if f(x) { + out = append(out, x) + } + } + return out +} + +// uniq returns a sorted slice containing the unique elements of list. +func uniq(list []string) []string { + out := make([]string, len(list)) + copy(out, list) + sort.Strings(out) + keep := out[:0] + for _, x := range out { + if len(keep) == 0 || keep[len(keep)-1] != x { + keep = append(keep, x) + } + } + return keep +} + +const ( + CheckExit = 1 << iota + ShowOutput + Background +) + +var outputLock sync.Mutex + +// run runs the command line cmd in dir. +// If mode has ShowOutput set and Background unset, run passes cmd's output to +// stdout/stderr directly. Otherwise, run returns cmd's output as a string. +// If mode has CheckExit set and the command fails, run calls fatalf. +// If mode has Background set, this command is being run as a +// Background job. Only bgrun should use the Background mode, +// not other callers. +func run(dir string, mode int, cmd ...string) string { + if vflag > 1 { + errprintf("run: %s\n", strings.Join(cmd, " ")) + } + + xcmd := exec.Command(cmd[0], cmd[1:]...) + setDir(xcmd, dir) + var data []byte + var err error + + // If we want to show command output and this is not + // a background command, assume it's the only thing + // running, so we can just let it write directly stdout/stderr + // as it runs without fear of mixing the output with some + // other command's output. Not buffering lets the output + // appear as it is printed instead of once the command exits. + // This is most important for the invocation of 'go1.4 build -v bootstrap/...'. + if mode&(Background|ShowOutput) == ShowOutput { + xcmd.Stdout = os.Stdout + xcmd.Stderr = os.Stderr + err = xcmd.Run() + } else { + data, err = xcmd.CombinedOutput() + } + if err != nil && mode&CheckExit != 0 { + outputLock.Lock() + if len(data) > 0 { + xprintf("%s\n", data) + } + outputLock.Unlock() + if mode&Background != 0 { + // Prevent fatalf from waiting on our own goroutine's + // bghelper to exit: + bghelpers.Done() + } + fatalf("FAILED: %v: %v", strings.Join(cmd, " "), err) + } + if mode&ShowOutput != 0 { + outputLock.Lock() + os.Stdout.Write(data) + outputLock.Unlock() + } + if vflag > 2 { + errprintf("run: %s DONE\n", strings.Join(cmd, " ")) + } + return string(data) +} + +var maxbg = 4 /* maximum number of jobs to run at once */ + +var ( + bgwork = make(chan func(), 1e5) + + bghelpers sync.WaitGroup + + dieOnce sync.Once // guards close of dying + dying = make(chan struct{}) +) + +func bginit() { + bghelpers.Add(maxbg) + for i := 0; i < maxbg; i++ { + go bghelper() + } +} + +func bghelper() { + defer bghelpers.Done() + for { + select { + case <-dying: + return + case w := <-bgwork: + // Dying takes precedence over doing more work. + select { + case <-dying: + return + default: + w() + } + } + } +} + +// bgrun is like run but runs the command in the background. +// CheckExit|ShowOutput mode is implied (since output cannot be returned). +// bgrun adds 1 to wg immediately, and calls Done when the work completes. +func bgrun(wg *sync.WaitGroup, dir string, cmd ...string) { + wg.Add(1) + bgwork <- func() { + defer wg.Done() + run(dir, CheckExit|ShowOutput|Background, cmd...) + } +} + +// bgwait waits for pending bgruns to finish. +// bgwait must be called from only a single goroutine at a time. +func bgwait(wg *sync.WaitGroup) { + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + select { + case <-done: + case <-dying: + // Don't return to the caller, to avoid reporting additional errors + // to the user. + select {} + } +} + +// xgetwd returns the current directory. +func xgetwd() string { + wd, err := os.Getwd() + if err != nil { + fatalf("%s", err) + } + return wd +} + +// xrealwd returns the 'real' name for the given path. +// real is defined as what xgetwd returns in that directory. +func xrealwd(path string) string { + old := xgetwd() + if err := os.Chdir(path); err != nil { + fatalf("chdir %s: %v", path, err) + } + real := xgetwd() + if err := os.Chdir(old); err != nil { + fatalf("chdir %s: %v", old, err) + } + return real +} + +// isdir reports whether p names an existing directory. +func isdir(p string) bool { + fi, err := os.Stat(p) + return err == nil && fi.IsDir() +} + +// isfile reports whether p names an existing file. +func isfile(p string) bool { + fi, err := os.Stat(p) + return err == nil && fi.Mode().IsRegular() +} + +// mtime returns the modification time of the file p. +func mtime(p string) time.Time { + fi, err := os.Stat(p) + if err != nil { + return time.Time{} + } + return fi.ModTime() +} + +// readfile returns the content of the named file. +func readfile(file string) string { + data, err := ioutil.ReadFile(file) + if err != nil { + fatalf("%v", err) + } + return string(data) +} + +const ( + writeExec = 1 << iota + writeSkipSame +) + +// writefile writes text to the named file, creating it if needed. +// if exec is non-zero, marks the file as executable. +// If the file already exists and has the expected content, +// it is not rewritten, to avoid changing the time stamp. +func writefile(text, file string, flag int) { + new := []byte(text) + if flag&writeSkipSame != 0 { + old, err := ioutil.ReadFile(file) + if err == nil && bytes.Equal(old, new) { + return + } + } + mode := os.FileMode(0666) + if flag&writeExec != 0 { + mode = 0777 + } + xremove(file) // in case of symlink tricks by misc/reboot test + err := ioutil.WriteFile(file, new, mode) + if err != nil { + fatalf("%v", err) + } +} + +// xmkdir creates the directory p. +func xmkdir(p string) { + err := os.Mkdir(p, 0777) + if err != nil { + fatalf("%v", err) + } +} + +// xmkdirall creates the directory p and its parents, as needed. +func xmkdirall(p string) { + err := os.MkdirAll(p, 0777) + if err != nil { + fatalf("%v", err) + } +} + +// xremove removes the file p. +func xremove(p string) { + if vflag > 2 { + errprintf("rm %s\n", p) + } + os.Remove(p) +} + +// xremoveall removes the file or directory tree rooted at p. +func xremoveall(p string) { + if vflag > 2 { + errprintf("rm -r %s\n", p) + } + os.RemoveAll(p) +} + +// xreaddir replaces dst with a list of the names of the files and subdirectories in dir. +// The names are relative to dir; they are not full paths. +func xreaddir(dir string) []string { + f, err := os.Open(dir) + if err != nil { + fatalf("%v", err) + } + defer f.Close() + names, err := f.Readdirnames(-1) + if err != nil { + fatalf("reading %s: %v", dir, err) + } + return names +} + +// xreaddir replaces dst with a list of the names of the files in dir. +// The names are relative to dir; they are not full paths. +func xreaddirfiles(dir string) []string { + f, err := os.Open(dir) + if err != nil { + fatalf("%v", err) + } + defer f.Close() + infos, err := f.Readdir(-1) + if err != nil { + fatalf("reading %s: %v", dir, err) + } + var names []string + for _, fi := range infos { + if !fi.IsDir() { + names = append(names, fi.Name()) + } + } + return names +} + +// xworkdir creates a new temporary directory to hold object files +// and returns the name of that directory. +func xworkdir() string { + name, err := ioutil.TempDir(os.Getenv("GOTMPDIR"), "go-tool-dist-") + if err != nil { + fatalf("%v", err) + } + return name +} + +// fatalf prints an error message to standard error and exits. +func fatalf(format string, args ...interface{}) { + fmt.Fprintf(os.Stderr, "go tool dist: %s\n", fmt.Sprintf(format, args...)) + + dieOnce.Do(func() { close(dying) }) + + // Wait for background goroutines to finish, + // so that exit handler that removes the work directory + // is not fighting with active writes or open files. + bghelpers.Wait() + + xexit(2) +} + +var atexits []func() + +// xexit exits the process with return code n. +func xexit(n int) { + for i := len(atexits) - 1; i >= 0; i-- { + atexits[i]() + } + os.Exit(n) +} + +// xatexit schedules the exit-handler f to be run when the program exits. +func xatexit(f func()) { + atexits = append(atexits, f) +} + +// xprintf prints a message to standard output. +func xprintf(format string, args ...interface{}) { + fmt.Printf(format, args...) +} + +// errprintf prints a message to standard output. +func errprintf(format string, args ...interface{}) { + fmt.Fprintf(os.Stderr, format, args...) +} + +// xsamefile reports whether f1 and f2 are the same file (or dir) +func xsamefile(f1, f2 string) bool { + fi1, err1 := os.Stat(f1) + fi2, err2 := os.Stat(f2) + if err1 != nil || err2 != nil { + return f1 == f2 + } + return os.SameFile(fi1, fi2) +} + +func xgetgoarm() string { + if goos == "android" { + // Assume all android devices have VFPv3. + // These ports are also mostly cross-compiled, so it makes little + // sense to auto-detect the setting. + return "7" + } + if goos == "windows" { + // windows/arm only works with ARMv7 executables. + return "7" + } + if gohostarch != "arm" || goos != gohostos { + // Conservative default for cross-compilation. + return "5" + } + + // Try to exec ourselves in a mode to detect VFP support. + // Seeing how far it gets determines which instructions failed. + // The test is OS-agnostic. + out := run("", 0, os.Args[0], "-check-goarm") + v1ok := strings.Contains(out, "VFPv1 OK.") + v3ok := strings.Contains(out, "VFPv3 OK.") + + if v1ok && v3ok { + return "7" + } + if v1ok { + return "6" + } + return "5" +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +// elfIsLittleEndian detects if the ELF file is little endian. +func elfIsLittleEndian(fn string) bool { + // read the ELF file header to determine the endianness without using the + // debug/elf package. + file, err := os.Open(fn) + if err != nil { + fatalf("failed to open file to determine endianness: %v", err) + } + defer file.Close() + var hdr [16]byte + if _, err := io.ReadFull(file, hdr[:]); err != nil { + fatalf("failed to read ELF header to determine endianness: %v", err) + } + // hdr[5] is EI_DATA byte, 1 is ELFDATA2LSB and 2 is ELFDATA2MSB + switch hdr[5] { + default: + fatalf("unknown ELF endianness of %s: EI_DATA = %d", fn, hdr[5]) + case 1: + return true + case 2: + return false + } + panic("unreachable") +} + +// count is a flag.Value that is like a flag.Bool and a flag.Int. +// If used as -name, it increments the count, but -name=x sets the count. +// Used for verbose flag -v. +type count int + +func (c *count) String() string { + return fmt.Sprint(int(*c)) +} + +func (c *count) Set(s string) error { + switch s { + case "true": + *c++ + case "false": + *c = 0 + default: + n, err := strconv.Atoi(s) + if err != nil { + return fmt.Errorf("invalid count %q", s) + } + *c = count(n) + } + return nil +} + +func (c *count) IsBoolFlag() bool { + return true +} + +func xflagparse(maxargs int) { + flag.Var((*count)(&vflag), "v", "verbosity") + flag.Parse() + if maxargs >= 0 && flag.NArg() > maxargs { + flag.Usage() + } +} diff --git a/src/cmd/dist/util_gc.go b/src/cmd/dist/util_gc.go new file mode 100644 index 0000000..875784d --- /dev/null +++ b/src/cmd/dist/util_gc.go @@ -0,0 +1,21 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build gc +// +build gc + +package main + +// useVFPv1 tries to execute one VFPv1 instruction on ARM. +// It will crash the current process if VFPv1 is missing. +func useVFPv1() + +// useVFPv3 tries to execute one VFPv3 instruction on ARM. +// It will crash the current process if VFPv3 is missing. +func useVFPv3() + +// useARMv6K tries to run ARMv6K instructions on ARM. +// It will crash the current process if it doesn't implement +// ARMv6K or above. +func useARMv6K() diff --git a/src/cmd/dist/util_gccgo.go b/src/cmd/dist/util_gccgo.go new file mode 100644 index 0000000..3255b80 --- /dev/null +++ b/src/cmd/dist/util_gccgo.go @@ -0,0 +1,14 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build gccgo +// +build gccgo + +package main + +func useVFPv1() {} + +func useVFPv3() {} + +func useARMv6K() {} diff --git a/src/cmd/dist/vfp_arm.s b/src/cmd/dist/vfp_arm.s new file mode 100644 index 0000000..525ee9b --- /dev/null +++ b/src/cmd/dist/vfp_arm.s @@ -0,0 +1,27 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build gc +// +build gc + +#include "textflag.h" + +// try to run "vmov.f64 d0, d0" instruction +TEXT ·useVFPv1(SB),NOSPLIT,$0 + WORD $0xeeb00b40 // vmov.f64 d0, d0 + RET + +// try to run VFPv3-only "vmov.f64 d0, #112" instruction +TEXT ·useVFPv3(SB),NOSPLIT,$0 + WORD $0xeeb70b00 // vmov.f64 d0, #112 + RET + +// try to run ARMv6K (or above) "ldrexd" instruction +TEXT ·useARMv6K(SB),NOSPLIT,$32 + MOVW R13, R2 + BIC $15, R13 + WORD $0xe1bd0f9f // ldrexd r0, r1, [sp] + WORD $0xf57ff01f // clrex + MOVW R2, R13 + RET diff --git a/src/cmd/dist/vfp_default.s b/src/cmd/dist/vfp_default.s new file mode 100644 index 0000000..0c1e16b --- /dev/null +++ b/src/cmd/dist/vfp_default.s @@ -0,0 +1,17 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build gc && !arm +// +build gc,!arm + +#include "textflag.h" + +TEXT ·useVFPv1(SB),NOSPLIT,$0 + RET + +TEXT ·useVFPv3(SB),NOSPLIT,$0 + RET + +TEXT ·useARMv6K(SB),NOSPLIT,$0 + RET |