diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-28 13:14:23 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-28 13:14:23 +0000 |
commit | 73df946d56c74384511a194dd01dbe099584fd1a (patch) | |
tree | fd0bcea490dd81327ddfbb31e215439672c9a068 /src/cmd/go/internal/modload | |
parent | Initial commit. (diff) | |
download | golang-1.16-73df946d56c74384511a194dd01dbe099584fd1a.tar.xz golang-1.16-73df946d56c74384511a194dd01dbe099584fd1a.zip |
Adding upstream version 1.16.10.upstream/1.16.10upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/cmd/go/internal/modload')
-rw-r--r-- | src/cmd/go/internal/modload/build.go | 361 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/buildlist.go | 308 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/help.go | 64 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/import.go | 600 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/import_test.go | 96 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/init.go | 1072 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/list.go | 191 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/load.go | 1364 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/modfile.go | 591 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/mvs.go | 113 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/mvs_test.go | 31 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/query.go | 1111 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/query_test.go | 216 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/search.go | 219 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/stat_openfile.go | 28 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/stat_unix.go | 32 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/stat_windows.go | 21 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/vendor.go | 219 |
18 files changed, 6637 insertions, 0 deletions
diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go new file mode 100644 index 0000000..8ad5f83 --- /dev/null +++ b/src/cmd/go/internal/modload/build.go @@ -0,0 +1,361 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "bytes" + "context" + "encoding/hex" + "errors" + "fmt" + "internal/goroot" + "os" + "path/filepath" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/modfetch" + "cmd/go/internal/modinfo" + "cmd/go/internal/search" + + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +var ( + infoStart, _ = hex.DecodeString("3077af0c9274080241e1c107e6d618e6") + infoEnd, _ = hex.DecodeString("f932433186182072008242104116d8f2") +) + +func isStandardImportPath(path string) bool { + return findStandardImportPath(path) != "" +} + +func findStandardImportPath(path string) string { + if path == "" { + panic("findStandardImportPath called with empty path") + } + if search.IsStandardImportPath(path) { + if goroot.IsStandardPackage(cfg.GOROOT, cfg.BuildContext.Compiler, path) { + return filepath.Join(cfg.GOROOT, "src", path) + } + } + return "" +} + +// PackageModuleInfo returns information about the module that provides +// a given package. If modules are not enabled or if the package is in the +// standard library or if the package was not successfully loaded with +// LoadPackages or ImportFromFiles, nil is returned. +func PackageModuleInfo(pkgpath string) *modinfo.ModulePublic { + if isStandardImportPath(pkgpath) || !Enabled() { + return nil + } + m, ok := findModule(pkgpath) + if !ok { + return nil + } + fromBuildList := true + listRetracted := false + return moduleInfo(context.TODO(), m, fromBuildList, listRetracted) +} + +func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic { + if !Enabled() { + return nil + } + + listRetracted := false + if i := strings.Index(path, "@"); i >= 0 { + m := module.Version{Path: path[:i], Version: path[i+1:]} + fromBuildList := false + return moduleInfo(ctx, m, fromBuildList, listRetracted) + } + + for _, m := range buildList { + if m.Path == path { + fromBuildList := true + return moduleInfo(ctx, m, fromBuildList, listRetracted) + } + } + + return &modinfo.ModulePublic{ + Path: path, + Error: &modinfo.ModuleError{ + Err: "module not in current build", + }, + } +} + +// addUpdate fills in m.Update if an updated version is available. +func addUpdate(ctx context.Context, m *modinfo.ModulePublic) { + if m.Version == "" { + return + } + + if info, err := Query(ctx, m.Path, "upgrade", m.Version, CheckAllowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { + m.Update = &modinfo.ModulePublic{ + Path: m.Path, + Version: info.Version, + Time: &info.Time, + } + } +} + +// addVersions fills in m.Versions with the list of known versions. +// Excluded versions will be omitted. If listRetracted is false, retracted +// versions will also be omitted. +func addVersions(ctx context.Context, m *modinfo.ModulePublic, listRetracted bool) { + allowed := CheckAllowed + if listRetracted { + allowed = CheckExclusions + } + m.Versions, _ = versions(ctx, m.Path, allowed) +} + +// addRetraction fills in m.Retracted if the module was retracted by its author. +// m.Error is set if there's an error loading retraction information. +func addRetraction(ctx context.Context, m *modinfo.ModulePublic) { + if m.Version == "" { + return + } + + err := CheckRetractions(ctx, module.Version{Path: m.Path, Version: m.Version}) + var rerr *ModuleRetractedError + if errors.As(err, &rerr) { + if len(rerr.Rationale) == 0 { + m.Retracted = []string{"retracted by module author"} + } else { + m.Retracted = rerr.Rationale + } + } else if err != nil && m.Error == nil { + m.Error = &modinfo.ModuleError{Err: err.Error()} + } +} + +func moduleInfo(ctx context.Context, m module.Version, fromBuildList, listRetracted bool) *modinfo.ModulePublic { + if m == Target { + info := &modinfo.ModulePublic{ + Path: m.Path, + Version: m.Version, + Main: true, + } + if HasModRoot() { + info.Dir = ModRoot() + info.GoMod = ModFilePath() + if modFile.Go != nil { + info.GoVersion = modFile.Go.Version + } + } + return info + } + + info := &modinfo.ModulePublic{ + Path: m.Path, + Version: m.Version, + Indirect: fromBuildList && loaded != nil && !loaded.direct[m.Path], + } + if v, ok := rawGoVersion.Load(m); ok { + info.GoVersion = v.(string) + } + + // completeFromModCache fills in the extra fields in m using the module cache. + completeFromModCache := func(m *modinfo.ModulePublic) { + mod := module.Version{Path: m.Path, Version: m.Version} + + if m.Version != "" { + if q, err := Query(ctx, m.Path, m.Version, "", nil); err != nil { + m.Error = &modinfo.ModuleError{Err: err.Error()} + } else { + m.Version = q.Version + m.Time = &q.Time + } + + gomod, err := modfetch.CachePath(mod, "mod") + if err == nil { + if info, err := os.Stat(gomod); err == nil && info.Mode().IsRegular() { + m.GoMod = gomod + } + } + dir, err := modfetch.DownloadDir(mod) + if err == nil { + m.Dir = dir + } + + if listRetracted { + addRetraction(ctx, m) + } + } + + if m.GoVersion == "" { + if summary, err := rawGoModSummary(mod); err == nil && summary.goVersionV != "" { + m.GoVersion = summary.goVersionV[1:] + } + } + } + + if !fromBuildList { + // If this was an explicitly-versioned argument to 'go mod download' or + // 'go list -m', report the actual requested version, not its replacement. + completeFromModCache(info) // Will set m.Error in vendor mode. + return info + } + + r := Replacement(m) + if r.Path == "" { + if cfg.BuildMod == "vendor" { + // It's tempting to fill in the "Dir" field to point within the vendor + // directory, but that would be misleading: the vendor directory contains + // a flattened package tree, not complete modules, and it can even + // interleave packages from different modules if one module path is a + // prefix of the other. + } else { + completeFromModCache(info) + } + return info + } + + // Don't hit the network to fill in extra data for replaced modules. + // The original resolved Version and Time don't matter enough to be + // worth the cost, and we're going to overwrite the GoMod and Dir from the + // replacement anyway. See https://golang.org/issue/27859. + info.Replace = &modinfo.ModulePublic{ + Path: r.Path, + Version: r.Version, + } + if v, ok := rawGoVersion.Load(m); ok { + info.Replace.GoVersion = v.(string) + } + if r.Version == "" { + if filepath.IsAbs(r.Path) { + info.Replace.Dir = r.Path + } else { + info.Replace.Dir = filepath.Join(ModRoot(), r.Path) + } + info.Replace.GoMod = filepath.Join(info.Replace.Dir, "go.mod") + } + if cfg.BuildMod != "vendor" { + completeFromModCache(info.Replace) + info.Dir = info.Replace.Dir + info.GoMod = info.Replace.GoMod + info.Retracted = info.Replace.Retracted + } + info.GoVersion = info.Replace.GoVersion + return info +} + +// PackageBuildInfo returns a string containing module version information +// for modules providing packages named by path and deps. path and deps must +// name packages that were resolved successfully with LoadPackages. +func PackageBuildInfo(path string, deps []string) string { + if isStandardImportPath(path) || !Enabled() { + return "" + } + + target := mustFindModule(path, path) + mdeps := make(map[module.Version]bool) + for _, dep := range deps { + if !isStandardImportPath(dep) { + mdeps[mustFindModule(path, dep)] = true + } + } + var mods []module.Version + delete(mdeps, target) + for mod := range mdeps { + mods = append(mods, mod) + } + module.Sort(mods) + + var buf bytes.Buffer + fmt.Fprintf(&buf, "path\t%s\n", path) + + writeEntry := func(token string, m module.Version) { + mv := m.Version + if mv == "" { + mv = "(devel)" + } + fmt.Fprintf(&buf, "%s\t%s\t%s", token, m.Path, mv) + if r := Replacement(m); r.Path == "" { + fmt.Fprintf(&buf, "\t%s\n", modfetch.Sum(m)) + } else { + fmt.Fprintf(&buf, "\n=>\t%s\t%s\t%s\n", r.Path, r.Version, modfetch.Sum(r)) + } + } + + writeEntry("mod", target) + for _, mod := range mods { + writeEntry("dep", mod) + } + + return buf.String() +} + +// mustFindModule is like findModule, but it calls base.Fatalf if the +// module can't be found. +// +// TODO(jayconrod): remove this. Callers should use findModule and return +// errors instead of relying on base.Fatalf. +func mustFindModule(target, path string) module.Version { + pkg, ok := loaded.pkgCache.Get(path).(*loadPkg) + if ok { + if pkg.err != nil { + base.Fatalf("build %v: cannot load %v: %v", target, path, pkg.err) + } + return pkg.mod + } + + if path == "command-line-arguments" { + return Target + } + + base.Fatalf("build %v: cannot find module for path %v", target, path) + panic("unreachable") +} + +// findModule searches for the module that contains the package at path. +// If the package was loaded, its containing module and true are returned. +// Otherwise, module.Version{} and false are returend. +func findModule(path string) (module.Version, bool) { + if pkg, ok := loaded.pkgCache.Get(path).(*loadPkg); ok { + return pkg.mod, pkg.mod != module.Version{} + } + if path == "command-line-arguments" { + return Target, true + } + return module.Version{}, false +} + +func ModInfoProg(info string, isgccgo bool) []byte { + // Inject a variable with the debug information as runtime.modinfo, + // but compile it in package main so that it is specific to the binary. + // The variable must be a literal so that it will have the correct value + // before the initializer for package main runs. + // + // The runtime startup code refers to the variable, which keeps it live + // in all binaries. + // + // Note: we use an alternate recipe below for gccgo (based on an + // init function) due to the fact that gccgo does not support + // applying a "//go:linkname" directive to a variable. This has + // drawbacks in that other packages may want to look at the module + // info in their init functions (see issue 29628), which won't + // work for gccgo. See also issue 30344. + + if !isgccgo { + return []byte(fmt.Sprintf(`package main +import _ "unsafe" +//go:linkname __debug_modinfo__ runtime.modinfo +var __debug_modinfo__ = %q +`, string(infoStart)+info+string(infoEnd))) + } else { + return []byte(fmt.Sprintf(`package main +import _ "unsafe" +//go:linkname __set_debug_modinfo__ runtime.setmodinfo +func __set_debug_modinfo__(string) +func init() { __set_debug_modinfo__(%q) } +`, string(infoStart)+info+string(infoEnd))) + } +} diff --git a/src/cmd/go/internal/modload/buildlist.go b/src/cmd/go/internal/modload/buildlist.go new file mode 100644 index 0000000..0ed9853 --- /dev/null +++ b/src/cmd/go/internal/modload/buildlist.go @@ -0,0 +1,308 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/imports" + "cmd/go/internal/mvs" + "context" + "fmt" + "go/build" + "os" + "strings" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +// buildList is the list of modules to use for building packages. +// It is initialized by calling LoadPackages or ImportFromFiles, +// each of which uses loaded.load. +// +// Ideally, exactly ONE of those functions would be called, +// and exactly once. Most of the time, that's true. +// During "go get" it may not be. TODO(rsc): Figure out if +// that restriction can be established, or else document why not. +// +var buildList []module.Version + +// additionalExplicitRequirements is a list of modules paths for which +// WriteGoMod should record explicit requirements, even if they would be +// selected without those requirements. Each path must also appear in buildList. +var additionalExplicitRequirements []string + +// capVersionSlice returns s with its cap reduced to its length. +func capVersionSlice(s []module.Version) []module.Version { + return s[:len(s):len(s)] +} + +// LoadAllModules loads and returns the list of modules matching the "all" +// module pattern, starting with the Target module and in a deterministic +// (stable) order, without loading any packages. +// +// Modules are loaded automatically (and lazily) in LoadPackages: +// LoadAllModules need only be called if LoadPackages is not, +// typically in commands that care about modules but no particular package. +// +// The caller must not modify the returned list, but may append to it. +func LoadAllModules(ctx context.Context) []module.Version { + LoadModFile(ctx) + ReloadBuildList() + WriteGoMod() + return capVersionSlice(buildList) +} + +// Selected returns the selected version of the module with the given path, or +// the empty string if the given module has no selected version +// (either because it is not required or because it is the Target module). +func Selected(path string) (version string) { + if path == Target.Path { + return "" + } + for _, m := range buildList { + if m.Path == path { + return m.Version + } + } + return "" +} + +// EditBuildList edits the global build list by first adding every module in add +// to the existing build list, then adjusting versions (and adding or removing +// requirements as needed) until every module in mustSelect is selected at the +// given version. +// +// (Note that the newly-added modules might not be selected in the resulting +// build list: they could be lower than existing requirements or conflict with +// versions in mustSelect.) +// +// If the versions listed in mustSelect are mutually incompatible (due to one of +// the listed modules requiring a higher version of another), EditBuildList +// returns a *ConstraintError and leaves the build list in its previous state. +func EditBuildList(ctx context.Context, add, mustSelect []module.Version) error { + var upgraded = capVersionSlice(buildList) + if len(add) > 0 { + // First, upgrade the build list with any additions. + // In theory we could just append the additions to the build list and let + // mvs.Downgrade take care of resolving the upgrades too, but the + // diagnostics from Upgrade are currently much better in case of errors. + var err error + upgraded, err = mvs.Upgrade(Target, &mvsReqs{buildList: upgraded}, add...) + if err != nil { + return err + } + } + + downgraded, err := mvs.Downgrade(Target, &mvsReqs{buildList: append(upgraded, mustSelect...)}, mustSelect...) + if err != nil { + return err + } + + final, err := mvs.Upgrade(Target, &mvsReqs{buildList: downgraded}, mustSelect...) + if err != nil { + return err + } + + selected := make(map[string]module.Version, len(final)) + for _, m := range final { + selected[m.Path] = m + } + inconsistent := false + for _, m := range mustSelect { + s, ok := selected[m.Path] + if !ok { + if m.Version != "none" { + panic(fmt.Sprintf("internal error: mvs.BuildList lost %v", m)) + } + continue + } + if s.Version != m.Version { + inconsistent = true + break + } + } + + if !inconsistent { + buildList = final + additionalExplicitRequirements = make([]string, 0, len(mustSelect)) + for _, m := range mustSelect { + if m.Version != "none" { + additionalExplicitRequirements = append(additionalExplicitRequirements, m.Path) + } + } + return nil + } + + // We overshot one or more of the modules in mustSelected, which means that + // Downgrade removed something in mustSelect because it conflicted with + // something else in mustSelect. + // + // Walk the requirement graph to find the conflict. + // + // TODO(bcmills): Ideally, mvs.Downgrade (or a replacement for it) would do + // this directly. + + reqs := &mvsReqs{buildList: final} + reason := map[module.Version]module.Version{} + for _, m := range mustSelect { + reason[m] = m + } + queue := mustSelect[:len(mustSelect):len(mustSelect)] + for len(queue) > 0 { + var m module.Version + m, queue = queue[0], queue[1:] + required, err := reqs.Required(m) + if err != nil { + return err + } + for _, r := range required { + if _, ok := reason[r]; !ok { + reason[r] = reason[m] + queue = append(queue, r) + } + } + } + + var conflicts []Conflict + for _, m := range mustSelect { + s, ok := selected[m.Path] + if !ok { + if m.Version != "none" { + panic(fmt.Sprintf("internal error: mvs.BuildList lost %v", m)) + } + continue + } + if s.Version != m.Version { + conflicts = append(conflicts, Conflict{ + Source: reason[s], + Dep: s, + Constraint: m, + }) + } + } + + return &ConstraintError{ + Conflicts: conflicts, + } +} + +// A ConstraintError describes inconsistent constraints in EditBuildList +type ConstraintError struct { + // Conflict lists the source of the conflict for each version in mustSelect + // that could not be selected due to the requirements of some other version in + // mustSelect. + Conflicts []Conflict +} + +func (e *ConstraintError) Error() string { + b := new(strings.Builder) + b.WriteString("version constraints conflict:") + for _, c := range e.Conflicts { + fmt.Fprintf(b, "\n\t%v requires %v, but %v is requested", c.Source, c.Dep, c.Constraint) + } + return b.String() +} + +// A Conflict documents that Source requires Dep, which conflicts with Constraint. +// (That is, Dep has the same module path as Constraint but a higher version.) +type Conflict struct { + Source module.Version + Dep module.Version + Constraint module.Version +} + +// ReloadBuildList resets the state of loaded packages, then loads and returns +// the build list set by EditBuildList. +func ReloadBuildList() []module.Version { + loaded = loadFromRoots(loaderParams{ + PackageOpts: PackageOpts{ + Tags: imports.Tags(), + }, + listRoots: func() []string { return nil }, + allClosesOverTests: index.allPatternClosesOverTests(), // but doesn't matter because the root list is empty. + }) + return capVersionSlice(buildList) +} + +// CheckTidyVersion reports an error to stderr if the Go version indicated by +// the go.mod file is not supported by this version of the 'go' command. +// +// If allowError is false, such an error terminates the program. +func CheckTidyVersion(ctx context.Context, allowError bool) { + LoadModFile(ctx) + if index.goVersionV == "" { + return + } + + tags := build.Default.ReleaseTags + maxGo := tags[len(tags)-1] + if !strings.HasPrefix(maxGo, "go") || !modfile.GoVersionRE.MatchString(maxGo[2:]) { + base.Fatalf("go: unrecognized go version %q", maxGo) + } + max := maxGo[2:] + + if semver.Compare(index.goVersionV, "v"+max) > 0 { + have := index.goVersionV[1:] + if allowError { + fmt.Fprintf(os.Stderr, "go mod tidy: go.mod file indicates go %s, but maximum supported version is %s\n", have, max) + } else { + base.Fatalf("go mod tidy: go.mod file indicates go %s, but maximum supported version is %s\n", have, max) + } + } +} + +// TidyBuildList trims the build list to the minimal requirements needed to +// retain the same versions of all packages from the preceding call to +// LoadPackages. +func TidyBuildList() { + used := map[module.Version]bool{Target: true} + for _, pkg := range loaded.pkgs { + used[pkg.mod] = true + } + + keep := []module.Version{Target} + var direct []string + for _, m := range buildList[1:] { + if used[m] { + keep = append(keep, m) + if loaded.direct[m.Path] { + direct = append(direct, m.Path) + } + } else if cfg.BuildV { + if _, ok := index.require[m]; ok { + fmt.Fprintf(os.Stderr, "unused %s\n", m.Path) + } + } + } + + min, err := mvs.Req(Target, direct, &mvsReqs{buildList: keep}) + if err != nil { + base.Fatalf("go: %v", err) + } + buildList = append([]module.Version{Target}, min...) +} + +// checkMultiplePaths verifies that a given module path is used as itself +// or as a replacement for another module, but not both at the same time. +// +// (See https://golang.org/issue/26607 and https://golang.org/issue/34650.) +func checkMultiplePaths() { + firstPath := make(map[module.Version]string, len(buildList)) + for _, mod := range buildList { + src := mod + if rep := Replacement(mod); rep.Path != "" { + src = rep + } + if prev, ok := firstPath[src]; !ok { + firstPath[src] = mod.Path + } else if prev != mod.Path { + base.Errorf("go: %s@%s used for two different module paths (%s and %s)", src.Path, src.Version, prev, mod.Path) + } + } + base.ExitIfErrors() +} diff --git a/src/cmd/go/internal/modload/help.go b/src/cmd/go/internal/modload/help.go new file mode 100644 index 0000000..fd39ddd --- /dev/null +++ b/src/cmd/go/internal/modload/help.go @@ -0,0 +1,64 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import "cmd/go/internal/base" + +var HelpModules = &base.Command{ + UsageLine: "modules", + Short: "modules, module versions, and more", + Long: ` +Modules are how Go manages dependencies. + +A module is a collection of packages that are released, versioned, and +distributed together. Modules may be downloaded directly from version control +repositories or from module proxy servers. + +For a series of tutorials on modules, see +https://golang.org/doc/tutorial/create-module. + +For a detailed reference on modules, see https://golang.org/ref/mod. + +By default, the go command may download modules from https://proxy.golang.org. +It may authenticate modules using the checksum database at +https://sum.golang.org. Both services are operated by the Go team at Google. +The privacy policies for these services are available at +https://proxy.golang.org/privacy and https://sum.golang.org/privacy, +respectively. + +The go command's download behavior may be configured using GOPROXY, GOSUMDB, +GOPRIVATE, and other environment variables. See 'go help environment' +and https://golang.org/ref/mod#private-module-privacy for more information. + `, +} + +var HelpGoMod = &base.Command{ + UsageLine: "go.mod", + Short: "the go.mod file", + Long: ` +A module version is defined by a tree of source files, with a go.mod +file in its root. When the go command is run, it looks in the current +directory and then successive parent directories to find the go.mod +marking the root of the main (current) module. + +The go.mod file format is described in detail at +https://golang.org/ref/mod#go-mod-file. + +To create a new go.mod file, use 'go help init'. For details see +'go help mod init' or https://golang.org/ref/mod#go-mod-init. + +To add missing module requirements or remove unneeded requirements, +use 'go mod tidy'. For details, see 'go help mod tidy' or +https://golang.org/ref/mod#go-mod-tidy. + +To add, upgrade, downgrade, or remove a specific module requirement, use +'go get'. For details, see 'go help module-get' or +https://golang.org/ref/mod#go-get. + +To make other changes or to parse go.mod as JSON for use by other tools, +use 'go mod edit'. See 'go help mod edit' or +https://golang.org/ref/mod#go-mod-edit. + `, +} diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go new file mode 100644 index 0000000..eda0568 --- /dev/null +++ b/src/cmd/go/internal/modload/import.go @@ -0,0 +1,600 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "context" + "errors" + "fmt" + "go/build" + "internal/goroot" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/modfetch" + "cmd/go/internal/par" + "cmd/go/internal/search" + + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +type ImportMissingError struct { + Path string + Module module.Version + QueryErr error + + // isStd indicates whether we would expect to find the package in the standard + // library. This is normally true for all dotless import paths, but replace + // directives can cause us to treat the replaced paths as also being in + // modules. + isStd bool + + // replaced the highest replaced version of the module where the replacement + // contains the package. replaced is only set if the replacement is unused. + replaced module.Version + + // newMissingVersion is set to a newer version of Module if one is present + // in the build list. When set, we can't automatically upgrade. + newMissingVersion string +} + +func (e *ImportMissingError) Error() string { + if e.Module.Path == "" { + if e.isStd { + return fmt.Sprintf("package %s is not in GOROOT (%s)", e.Path, filepath.Join(cfg.GOROOT, "src", e.Path)) + } + if e.QueryErr != nil && e.QueryErr != ErrNoModRoot { + return fmt.Sprintf("cannot find module providing package %s: %v", e.Path, e.QueryErr) + } + if cfg.BuildMod == "mod" || (cfg.BuildMod == "readonly" && allowMissingModuleImports) { + return "cannot find module providing package " + e.Path + } + + if e.replaced.Path != "" { + suggestArg := e.replaced.Path + if !modfetch.IsZeroPseudoVersion(e.replaced.Version) { + suggestArg = e.replaced.String() + } + return fmt.Sprintf("module %s provides package %s and is replaced but not required; to add it:\n\tgo get %s", e.replaced.Path, e.Path, suggestArg) + } + + message := fmt.Sprintf("no required module provides package %s", e.Path) + if e.QueryErr != nil { + return fmt.Sprintf("%s: %v", message, e.QueryErr) + } + return fmt.Sprintf("%s; to add it:\n\tgo get %s", message, e.Path) + } + + if e.newMissingVersion != "" { + return fmt.Sprintf("package %s provided by %s at latest version %s but not at required version %s", e.Path, e.Module.Path, e.Module.Version, e.newMissingVersion) + } + + return fmt.Sprintf("missing module for import: %s@%s provides %s", e.Module.Path, e.Module.Version, e.Path) +} + +func (e *ImportMissingError) Unwrap() error { + return e.QueryErr +} + +func (e *ImportMissingError) ImportPath() string { + return e.Path +} + +// An AmbiguousImportError indicates an import of a package found in multiple +// modules in the build list, or found in both the main module and its vendor +// directory. +type AmbiguousImportError struct { + importPath string + Dirs []string + Modules []module.Version // Either empty or 1:1 with Dirs. +} + +func (e *AmbiguousImportError) ImportPath() string { + return e.importPath +} + +func (e *AmbiguousImportError) Error() string { + locType := "modules" + if len(e.Modules) == 0 { + locType = "directories" + } + + var buf strings.Builder + fmt.Fprintf(&buf, "ambiguous import: found package %s in multiple %s:", e.importPath, locType) + + for i, dir := range e.Dirs { + buf.WriteString("\n\t") + if i < len(e.Modules) { + m := e.Modules[i] + buf.WriteString(m.Path) + if m.Version != "" { + fmt.Fprintf(&buf, " %s", m.Version) + } + fmt.Fprintf(&buf, " (%s)", dir) + } else { + buf.WriteString(dir) + } + } + + return buf.String() +} + +// ImportMissingSumError is reported in readonly mode when we need to check +// if a module contains a package, but we don't have a sum for its .zip file. +// We might need sums for multiple modules to verify the package is unique. +// +// TODO(#43653): consolidate multiple errors of this type into a single error +// that suggests a 'go get' command for root packages that transtively import +// packages from modules with missing sums. load.CheckPackageErrors would be +// a good place to consolidate errors, but we'll need to attach the import +// stack here. +type ImportMissingSumError struct { + importPath string + found bool + mods []module.Version + importer, importerVersion string // optional, but used for additional context + importerIsTest bool +} + +func (e *ImportMissingSumError) Error() string { + var importParen string + if e.importer != "" { + importParen = fmt.Sprintf(" (imported by %s)", e.importer) + } + var message string + if e.found { + message = fmt.Sprintf("missing go.sum entry needed to verify package %s%s is provided by exactly one module", e.importPath, importParen) + } else { + message = fmt.Sprintf("missing go.sum entry for module providing package %s%s", e.importPath, importParen) + } + var hint string + if e.importer == "" { + // Importing package is unknown, or the missing package was named on the + // command line. Recommend 'go mod download' for the modules that could + // provide the package, since that shouldn't change go.mod. + if len(e.mods) > 0 { + args := make([]string, len(e.mods)) + for i, mod := range e.mods { + args[i] = mod.Path + } + hint = fmt.Sprintf("; to add:\n\tgo mod download %s", strings.Join(args, " ")) + } + } else { + // Importing package is known (common case). Recommend 'go get' on the + // current version of the importing package. + tFlag := "" + if e.importerIsTest { + tFlag = " -t" + } + version := "" + if e.importerVersion != "" { + version = "@" + e.importerVersion + } + hint = fmt.Sprintf("; to add:\n\tgo get%s %s%s", tFlag, e.importer, version) + } + return message + hint +} + +func (e *ImportMissingSumError) ImportPath() string { + return e.importPath +} + +type invalidImportError struct { + importPath string + err error +} + +func (e *invalidImportError) ImportPath() string { + return e.importPath +} + +func (e *invalidImportError) Error() string { + return e.err.Error() +} + +func (e *invalidImportError) Unwrap() error { + return e.err +} + +// importFromBuildList finds the module and directory in the build list +// containing the package with the given import path. The answer must be unique: +// importFromBuildList returns an error if multiple modules attempt to provide +// the same package. +// +// importFromBuildList can return a module with an empty m.Path, for packages in +// the standard library. +// +// importFromBuildList can return an empty directory string, for fake packages +// like "C" and "unsafe". +// +// If the package cannot be found in buildList, +// importFromBuildList returns an *ImportMissingError. +func importFromBuildList(ctx context.Context, path string, buildList []module.Version) (m module.Version, dir string, err error) { + if strings.Contains(path, "@") { + return module.Version{}, "", fmt.Errorf("import path should not have @version") + } + if build.IsLocalImport(path) { + return module.Version{}, "", fmt.Errorf("relative import not supported") + } + if path == "C" || path == "unsafe" { + // There's no directory for import "C" or import "unsafe". + return module.Version{}, "", nil + } + // Before any further lookup, check that the path is valid. + if err := module.CheckImportPath(path); err != nil { + return module.Version{}, "", &invalidImportError{importPath: path, err: err} + } + + // Is the package in the standard library? + pathIsStd := search.IsStandardImportPath(path) + if pathIsStd && goroot.IsStandardPackage(cfg.GOROOT, cfg.BuildContext.Compiler, path) { + if targetInGorootSrc { + if dir, ok, err := dirInModule(path, targetPrefix, ModRoot(), true); err != nil { + return module.Version{}, dir, err + } else if ok { + return Target, dir, nil + } + } + dir := filepath.Join(cfg.GOROOT, "src", path) + return module.Version{}, dir, nil + } + + // -mod=vendor is special. + // Everything must be in the main module or the main module's vendor directory. + if cfg.BuildMod == "vendor" { + mainDir, mainOK, mainErr := dirInModule(path, targetPrefix, ModRoot(), true) + vendorDir, vendorOK, _ := dirInModule(path, "", filepath.Join(ModRoot(), "vendor"), false) + if mainOK && vendorOK { + return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: []string{mainDir, vendorDir}} + } + // Prefer to return main directory if there is one, + // Note that we're not checking that the package exists. + // We'll leave that for load. + if !vendorOK && mainDir != "" { + return Target, mainDir, nil + } + if mainErr != nil { + return module.Version{}, "", mainErr + } + readVendorList() + return vendorPkgModule[path], vendorDir, nil + } + + // Check each module on the build list. + var dirs []string + var mods []module.Version + var sumErrMods []module.Version + for _, m := range buildList { + if !maybeInModule(path, m.Path) { + // Avoid possibly downloading irrelevant modules. + continue + } + needSum := true + root, isLocal, err := fetch(ctx, m, needSum) + if err != nil { + if sumErr := (*sumMissingError)(nil); errors.As(err, &sumErr) { + // We are missing a sum needed to fetch a module in the build list. + // We can't verify that the package is unique, and we may not find + // the package at all. Keep checking other modules to decide which + // error to report. Multiple sums may be missing if we need to look in + // multiple nested modules to resolve the import. + sumErrMods = append(sumErrMods, m) + continue + } + // Report fetch error. + // Note that we don't know for sure this module is necessary, + // but it certainly _could_ provide the package, and even if we + // continue the loop and find the package in some other module, + // we need to look at this module to make sure the import is + // not ambiguous. + return module.Version{}, "", err + } + if dir, ok, err := dirInModule(path, m.Path, root, isLocal); err != nil { + return module.Version{}, "", err + } else if ok { + mods = append(mods, m) + dirs = append(dirs, dir) + } + } + if len(mods) > 1 { + return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods} + } + if len(sumErrMods) > 0 { + return module.Version{}, "", &ImportMissingSumError{ + importPath: path, + mods: sumErrMods, + found: len(mods) > 0, + } + } + if len(mods) == 1 { + return mods[0], dirs[0], nil + } + + var queryErr error + if !HasModRoot() { + queryErr = ErrNoModRoot + } + return module.Version{}, "", &ImportMissingError{Path: path, QueryErr: queryErr, isStd: pathIsStd} +} + +// queryImport attempts to locate a module that can be added to the current +// build list to provide the package with the given import path. +// +// Unlike QueryPattern, queryImport prefers to add a replaced version of a +// module *before* checking the proxies for a version to add. +func queryImport(ctx context.Context, path string) (module.Version, error) { + // To avoid spurious remote fetches, try the latest replacement for each + // module (golang.org/issue/26241). + if index != nil { + var mods []module.Version + for mp, mv := range index.highestReplaced { + if !maybeInModule(path, mp) { + continue + } + if mv == "" { + // The only replacement is a wildcard that doesn't specify a version, so + // synthesize a pseudo-version with an appropriate major version and a + // timestamp below any real timestamp. That way, if the main module is + // used from within some other module, the user will be able to upgrade + // the requirement to any real version they choose. + if _, pathMajor, ok := module.SplitPathVersion(mp); ok && len(pathMajor) > 0 { + mv = modfetch.ZeroPseudoVersion(pathMajor[1:]) + } else { + mv = modfetch.ZeroPseudoVersion("v0") + } + } + mods = append(mods, module.Version{Path: mp, Version: mv}) + } + + // Every module path in mods is a prefix of the import path. + // As in QueryPattern, prefer the longest prefix that satisfies the import. + sort.Slice(mods, func(i, j int) bool { + return len(mods[i].Path) > len(mods[j].Path) + }) + for _, m := range mods { + needSum := true + root, isLocal, err := fetch(ctx, m, needSum) + if err != nil { + if sumErr := (*sumMissingError)(nil); errors.As(err, &sumErr) { + return module.Version{}, &ImportMissingSumError{importPath: path} + } + return module.Version{}, err + } + if _, ok, err := dirInModule(path, m.Path, root, isLocal); err != nil { + return m, err + } else if ok { + if cfg.BuildMod == "readonly" { + return module.Version{}, &ImportMissingError{Path: path, replaced: m} + } + return m, nil + } + } + if len(mods) > 0 && module.CheckPath(path) != nil { + // The package path is not valid to fetch remotely, + // so it can only exist in a replaced module, + // and we know from the above loop that it is not. + return module.Version{}, &PackageNotInModuleError{ + Mod: mods[0], + Query: "latest", + Pattern: path, + Replacement: Replacement(mods[0]), + } + } + } + + if search.IsStandardImportPath(path) { + // This package isn't in the standard library, isn't in any module already + // in the build list, and isn't in any other module that the user has + // shimmed in via a "replace" directive. + // Moreover, the import path is reserved for the standard library, so + // QueryPattern cannot possibly find a module containing this package. + // + // Instead of trying QueryPattern, report an ImportMissingError immediately. + return module.Version{}, &ImportMissingError{Path: path, isStd: true} + } + + if cfg.BuildMod == "readonly" && !allowMissingModuleImports { + // In readonly mode, we can't write go.mod, so we shouldn't try to look up + // the module. If readonly mode was enabled explicitly, include that in + // the error message. + var queryErr error + if cfg.BuildModExplicit { + queryErr = fmt.Errorf("import lookup disabled by -mod=%s", cfg.BuildMod) + } else if cfg.BuildModReason != "" { + queryErr = fmt.Errorf("import lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) + } + return module.Version{}, &ImportMissingError{Path: path, QueryErr: queryErr} + } + + // Look up module containing the package, for addition to the build list. + // Goal is to determine the module, download it to dir, + // and return m, dir, ImpportMissingError. + fmt.Fprintf(os.Stderr, "go: finding module for package %s\n", path) + + candidates, err := QueryPackages(ctx, path, "latest", Selected, CheckAllowed) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // Return "cannot find module providing package […]" instead of whatever + // low-level error QueryPattern produced. + return module.Version{}, &ImportMissingError{Path: path, QueryErr: err} + } else { + return module.Version{}, err + } + } + + candidate0MissingVersion := "" + for i, c := range candidates { + cm := c.Mod + canAdd := true + for _, bm := range buildList { + if bm.Path == cm.Path && semver.Compare(bm.Version, cm.Version) > 0 { + // QueryPattern proposed that we add module cm to provide the package, + // but we already depend on a newer version of that module (and we don't + // have the package). + // + // This typically happens when a package is present at the "@latest" + // version (e.g., v1.0.0) of a module, but we have a newer version + // of the same module in the build list (e.g., v1.0.1-beta), and + // the package is not present there. + canAdd = false + if i == 0 { + candidate0MissingVersion = bm.Version + } + break + } + } + if canAdd { + return cm, nil + } + } + return module.Version{}, &ImportMissingError{ + Path: path, + Module: candidates[0].Mod, + newMissingVersion: candidate0MissingVersion, + } +} + +// maybeInModule reports whether, syntactically, +// a package with the given import path could be supplied +// by a module with the given module path (mpath). +func maybeInModule(path, mpath string) bool { + return mpath == path || + len(path) > len(mpath) && path[len(mpath)] == '/' && path[:len(mpath)] == mpath +} + +var ( + haveGoModCache par.Cache // dir → bool + haveGoFilesCache par.Cache // dir → goFilesEntry +) + +type goFilesEntry struct { + haveGoFiles bool + err error +} + +// dirInModule locates the directory that would hold the package named by the given path, +// if it were in the module with module path mpath and root mdir. +// If path is syntactically not within mpath, +// or if mdir is a local file tree (isLocal == true) and the directory +// that would hold path is in a sub-module (covered by a go.mod below mdir), +// dirInModule returns "", false, nil. +// +// Otherwise, dirInModule returns the name of the directory where +// Go source files would be expected, along with a boolean indicating +// whether there are in fact Go source files in that directory. +// A non-nil error indicates that the existence of the directory and/or +// source files could not be determined, for example due to a permission error. +func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFiles bool, err error) { + // Determine where to expect the package. + if path == mpath { + dir = mdir + } else if mpath == "" { // vendor directory + dir = filepath.Join(mdir, path) + } else if len(path) > len(mpath) && path[len(mpath)] == '/' && path[:len(mpath)] == mpath { + dir = filepath.Join(mdir, path[len(mpath)+1:]) + } else { + return "", false, nil + } + + // Check that there aren't other modules in the way. + // This check is unnecessary inside the module cache + // and important to skip in the vendor directory, + // where all the module trees have been overlaid. + // So we only check local module trees + // (the main module, and any directory trees pointed at by replace directives). + if isLocal { + for d := dir; d != mdir && len(d) > len(mdir); { + haveGoMod := haveGoModCache.Do(d, func() interface{} { + fi, err := fsys.Stat(filepath.Join(d, "go.mod")) + return err == nil && !fi.IsDir() + }).(bool) + + if haveGoMod { + return "", false, nil + } + parent := filepath.Dir(d) + if parent == d { + // Break the loop, as otherwise we'd loop + // forever if d=="." and mdir=="". + break + } + d = parent + } + } + + // Now committed to returning dir (not ""). + + // Are there Go source files in the directory? + // We don't care about build tags, not even "+build ignore". + // We're just looking for a plausible directory. + res := haveGoFilesCache.Do(dir, func() interface{} { + ok, err := fsys.IsDirWithGoFiles(dir) + return goFilesEntry{haveGoFiles: ok, err: err} + }).(goFilesEntry) + + return dir, res.haveGoFiles, res.err +} + +// fetch downloads the given module (or its replacement) +// and returns its location. +// +// needSum indicates whether the module may be downloaded in readonly mode +// without a go.sum entry. It should only be false for modules fetched +// speculatively (for example, for incompatible version filtering). The sum +// will still be verified normally. +// +// The isLocal return value reports whether the replacement, +// if any, is local to the filesystem. +func fetch(ctx context.Context, mod module.Version, needSum bool) (dir string, isLocal bool, err error) { + if mod == Target { + return ModRoot(), true, nil + } + if r := Replacement(mod); r.Path != "" { + if r.Version == "" { + dir = r.Path + if !filepath.IsAbs(dir) { + dir = filepath.Join(ModRoot(), dir) + } + // Ensure that the replacement directory actually exists: + // dirInModule does not report errors for missing modules, + // so if we don't report the error now, later failures will be + // very mysterious. + if _, err := fsys.Stat(dir); err != nil { + if os.IsNotExist(err) { + // Semantically the module version itself “exists” — we just don't + // have its source code. Remove the equivalence to os.ErrNotExist, + // and make the message more concise while we're at it. + err = fmt.Errorf("replacement directory %s does not exist", r.Path) + } else { + err = fmt.Errorf("replacement directory %s: %w", r.Path, err) + } + return dir, true, module.VersionError(mod, err) + } + return dir, true, nil + } + mod = r + } + + if HasModRoot() && cfg.BuildMod == "readonly" && needSum && !modfetch.HaveSum(mod) { + return "", false, module.VersionError(mod, &sumMissingError{}) + } + + dir, err = modfetch.Download(ctx, mod) + return dir, false, err +} + +type sumMissingError struct { + suggestion string +} + +func (e *sumMissingError) Error() string { + return "missing go.sum entry" + e.suggestion +} diff --git a/src/cmd/go/internal/modload/import_test.go b/src/cmd/go/internal/modload/import_test.go new file mode 100644 index 0000000..9420dc5 --- /dev/null +++ b/src/cmd/go/internal/modload/import_test.go @@ -0,0 +1,96 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "context" + "internal/testenv" + "regexp" + "strings" + "testing" + + "golang.org/x/mod/module" +) + +var importTests = []struct { + path string + m module.Version + err string +}{ + { + path: "golang.org/x/net/context", + m: module.Version{ + Path: "golang.org/x/net", + }, + }, + { + path: "golang.org/x/net", + err: `module golang.org/x/net@.* found \(v0.0.0-.*\), but does not contain package golang.org/x/net`, + }, + { + path: "golang.org/x/text", + m: module.Version{ + Path: "golang.org/x/text", + }, + }, + { + path: "github.com/rsc/quote/buggy", + m: module.Version{ + Path: "github.com/rsc/quote", + Version: "v1.5.2", + }, + }, + { + path: "github.com/rsc/quote", + m: module.Version{ + Path: "github.com/rsc/quote", + Version: "v1.5.2", + }, + }, + { + path: "golang.org/x/foo/bar", + err: "cannot find module providing package golang.org/x/foo/bar", + }, +} + +func TestQueryImport(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + testenv.MustHaveExecPath(t, "git") + + oldAllowMissingModuleImports := allowMissingModuleImports + oldRootMode := RootMode + defer func() { + allowMissingModuleImports = oldAllowMissingModuleImports + RootMode = oldRootMode + }() + allowMissingModuleImports = true + RootMode = NoRoot + + ctx := context.Background() + + for _, tt := range importTests { + t.Run(strings.ReplaceAll(tt.path, "/", "_"), func(t *testing.T) { + // Note that there is no build list, so Import should always fail. + m, err := queryImport(ctx, tt.path) + + if tt.err == "" { + if err != nil { + t.Fatalf("queryImport(_, %q): %v", tt.path, err) + } + } else { + if err == nil { + t.Fatalf("queryImport(_, %q) = %v, nil; expected error", tt.path, m) + } + if !regexp.MustCompile(tt.err).MatchString(err.Error()) { + t.Fatalf("queryImport(_, %q): error %q, want error matching %#q", tt.path, err, tt.err) + } + } + + if m.Path != tt.m.Path || (tt.m.Version != "" && m.Version != tt.m.Version) { + t.Errorf("queryImport(_, %q) = %v, _; want %v", tt.path, m, tt.m) + } + }) + } +} diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go new file mode 100644 index 0000000..8ec1c86 --- /dev/null +++ b/src/cmd/go/internal/modload/init.go @@ -0,0 +1,1072 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "go/build" + "internal/lazyregexp" + "os" + "path" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/lockedfile" + "cmd/go/internal/modconv" + "cmd/go/internal/modfetch" + "cmd/go/internal/mvs" + "cmd/go/internal/search" + "cmd/go/internal/str" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +var ( + initialized bool + + modRoot string + Target module.Version + + // targetPrefix is the path prefix for packages in Target, without a trailing + // slash. For most modules, targetPrefix is just Target.Path, but the + // standard-library module "std" has an empty prefix. + targetPrefix string + + // targetInGorootSrc caches whether modRoot is within GOROOT/src. + // The "std" module is special within GOROOT/src, but not otherwise. + targetInGorootSrc bool + + gopath string + + // RootMode determines whether a module root is needed. + RootMode Root + + // ForceUseModules may be set to force modules to be enabled when + // GO111MODULE=auto or to report an error when GO111MODULE=off. + ForceUseModules bool + + allowMissingModuleImports bool +) + +type Root int + +const ( + // AutoRoot is the default for most commands. modload.Init will look for + // a go.mod file in the current directory or any parent. If none is found, + // modules may be disabled (GO111MODULE=on) or commands may run in a + // limited module mode. + AutoRoot Root = iota + + // NoRoot is used for commands that run in module mode and ignore any go.mod + // file the current directory or in parent directories. + NoRoot + + // NeedRoot is used for commands that must run in module mode and don't + // make sense without a main module. + NeedRoot +) + +// ModFile returns the parsed go.mod file. +// +// Note that after calling LoadPackages or LoadAllModules, +// the require statements in the modfile.File are no longer +// the source of truth and will be ignored: edits made directly +// will be lost at the next call to WriteGoMod. +// To make permanent changes to the require statements +// in go.mod, edit it before loading. +func ModFile() *modfile.File { + Init() + if modFile == nil { + die() + } + return modFile +} + +func BinDir() string { + Init() + return filepath.Join(gopath, "bin") +} + +// Init determines whether module mode is enabled, locates the root of the +// current module (if any), sets environment variables for Git subprocesses, and +// configures the cfg, codehost, load, modfetch, and search packages for use +// with modules. +func Init() { + if initialized { + return + } + initialized = true + + // Keep in sync with WillBeEnabled. We perform extra validation here, and + // there are lots of diagnostics and side effects, so we can't use + // WillBeEnabled directly. + var mustUseModules bool + env := cfg.Getenv("GO111MODULE") + switch env { + default: + base.Fatalf("go: unknown environment setting GO111MODULE=%s", env) + case "auto": + mustUseModules = ForceUseModules + case "on", "": + mustUseModules = true + case "off": + if ForceUseModules { + base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") + } + mustUseModules = false + return + } + + if err := fsys.Init(base.Cwd); err != nil { + base.Fatalf("go: %v", err) + } + + // Disable any prompting for passwords by Git. + // Only has an effect for 2.3.0 or later, but avoiding + // the prompt in earlier versions is just too hard. + // If user has explicitly set GIT_TERMINAL_PROMPT=1, keep + // prompting. + // See golang.org/issue/9341 and golang.org/issue/12706. + if os.Getenv("GIT_TERMINAL_PROMPT") == "" { + os.Setenv("GIT_TERMINAL_PROMPT", "0") + } + + // Disable any ssh connection pooling by Git. + // If a Git subprocess forks a child into the background to cache a new connection, + // that child keeps stdout/stderr open. After the Git subprocess exits, + // os /exec expects to be able to read from the stdout/stderr pipe + // until EOF to get all the data that the Git subprocess wrote before exiting. + // The EOF doesn't come until the child exits too, because the child + // is holding the write end of the pipe. + // This is unfortunate, but it has come up at least twice + // (see golang.org/issue/13453 and golang.org/issue/16104) + // and confuses users when it does. + // If the user has explicitly set GIT_SSH or GIT_SSH_COMMAND, + // assume they know what they are doing and don't step on it. + // But default to turning off ControlMaster. + if os.Getenv("GIT_SSH") == "" && os.Getenv("GIT_SSH_COMMAND") == "" { + os.Setenv("GIT_SSH_COMMAND", "ssh -o ControlMaster=no") + } + + if modRoot != "" { + // modRoot set before Init was called ("go mod init" does this). + // No need to search for go.mod. + } else if RootMode == NoRoot { + if cfg.ModFile != "" && !base.InGOFLAGS("-modfile") { + base.Fatalf("go: -modfile cannot be used with commands that ignore the current module") + } + modRoot = "" + } else { + modRoot = findModuleRoot(base.Cwd) + if modRoot == "" { + if cfg.ModFile != "" { + base.Fatalf("go: cannot find main module, but -modfile was set.\n\t-modfile cannot be used to set the module root directory.") + } + if RootMode == NeedRoot { + base.Fatalf("go: %v", ErrNoModRoot) + } + if !mustUseModules { + // GO111MODULE is 'auto', and we can't find a module root. + // Stay in GOPATH mode. + return + } + } else if search.InDir(modRoot, os.TempDir()) == "." { + // If you create /tmp/go.mod for experimenting, + // then any tests that create work directories under /tmp + // will find it and get modules when they're not expecting them. + // It's a bit of a peculiar thing to disallow but quite mysterious + // when it happens. See golang.org/issue/26708. + modRoot = "" + fmt.Fprintf(os.Stderr, "go: warning: ignoring go.mod in system temp root %v\n", os.TempDir()) + if !mustUseModules { + return + } + } + } + if cfg.ModFile != "" && !strings.HasSuffix(cfg.ModFile, ".mod") { + base.Fatalf("go: -modfile=%s: file does not have .mod extension", cfg.ModFile) + } + + // We're in module mode. Set any global variables that need to be set. + cfg.ModulesEnabled = true + setDefaultBuildMod() + list := filepath.SplitList(cfg.BuildContext.GOPATH) + if len(list) == 0 || list[0] == "" { + base.Fatalf("missing $GOPATH") + } + gopath = list[0] + if _, err := fsys.Stat(filepath.Join(gopath, "go.mod")); err == nil { + base.Fatalf("$GOPATH/go.mod exists but should not") + } + + if modRoot == "" { + // We're in module mode, but not inside a module. + // + // Commands like 'go build', 'go run', 'go list' have no go.mod file to + // read or write. They would need to find and download the latest versions + // of a potentially large number of modules with no way to save version + // information. We can succeed slowly (but not reproducibly), but that's + // not usually a good experience. + // + // Instead, we forbid resolving import paths to modules other than std and + // cmd. Users may still build packages specified with .go files on the + // command line, but they'll see an error if those files import anything + // outside std. + // + // This can be overridden by calling AllowMissingModuleImports. + // For example, 'go get' does this, since it is expected to resolve paths. + // + // See golang.org/issue/32027. + } else { + modfetch.GoSumFile = strings.TrimSuffix(ModFilePath(), ".mod") + ".sum" + search.SetModRoot(modRoot) + } +} + +// WillBeEnabled checks whether modules should be enabled but does not +// initialize modules by installing hooks. If Init has already been called, +// WillBeEnabled returns the same result as Enabled. +// +// This function is needed to break a cycle. The main package needs to know +// whether modules are enabled in order to install the module or GOPATH version +// of 'go get', but Init reads the -modfile flag in 'go get', so it shouldn't +// be called until the command is installed and flags are parsed. Instead of +// calling Init and Enabled, the main package can call this function. +func WillBeEnabled() bool { + if modRoot != "" || cfg.ModulesEnabled { + // Already enabled. + return true + } + if initialized { + // Initialized, not enabled. + return false + } + + // Keep in sync with Init. Init does extra validation and prints warnings or + // exits, so it can't call this function directly. + env := cfg.Getenv("GO111MODULE") + switch env { + case "on", "": + return true + case "auto": + break + default: + return false + } + + if modRoot := findModuleRoot(base.Cwd); modRoot == "" { + // GO111MODULE is 'auto', and we can't find a module root. + // Stay in GOPATH mode. + return false + } else if search.InDir(modRoot, os.TempDir()) == "." { + // If you create /tmp/go.mod for experimenting, + // then any tests that create work directories under /tmp + // will find it and get modules when they're not expecting them. + // It's a bit of a peculiar thing to disallow but quite mysterious + // when it happens. See golang.org/issue/26708. + return false + } + return true +} + +// Enabled reports whether modules are (or must be) enabled. +// If modules are enabled but there is no main module, Enabled returns true +// and then the first use of module information will call die +// (usually through MustModRoot). +func Enabled() bool { + Init() + return modRoot != "" || cfg.ModulesEnabled +} + +// ModRoot returns the root of the main module. +// It calls base.Fatalf if there is no main module. +func ModRoot() string { + if !HasModRoot() { + die() + } + return modRoot +} + +// HasModRoot reports whether a main module is present. +// HasModRoot may return false even if Enabled returns true: for example, 'get' +// does not require a main module. +func HasModRoot() bool { + Init() + return modRoot != "" +} + +// ModFilePath returns the effective path of the go.mod file. Normally, this +// "go.mod" in the directory returned by ModRoot, but the -modfile flag may +// change its location. ModFilePath calls base.Fatalf if there is no main +// module, even if -modfile is set. +func ModFilePath() string { + if !HasModRoot() { + die() + } + if cfg.ModFile != "" { + return cfg.ModFile + } + return filepath.Join(modRoot, "go.mod") +} + +func die() { + if cfg.Getenv("GO111MODULE") == "off" { + base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") + } + if dir, name := findAltConfig(base.Cwd); dir != "" { + rel, err := filepath.Rel(base.Cwd, dir) + if err != nil { + rel = dir + } + cdCmd := "" + if rel != "." { + cdCmd = fmt.Sprintf("cd %s && ", rel) + } + base.Fatalf("go: cannot find main module, but found %s in %s\n\tto create a module there, run:\n\t%sgo mod init", name, dir, cdCmd) + } + base.Fatalf("go: %v", ErrNoModRoot) +} + +var ErrNoModRoot = errors.New("go.mod file not found in current directory or any parent directory; see 'go help modules'") + +// LoadModFile sets Target and, if there is a main module, parses the initial +// build list from its go.mod file. +// +// LoadModFile may make changes in memory, like adding a go directive and +// ensuring requirements are consistent. WriteGoMod should be called later to +// write changes out to disk or report errors in readonly mode. +// +// As a side-effect, LoadModFile may change cfg.BuildMod to "vendor" if +// -mod wasn't set explicitly and automatic vendoring should be enabled. +func LoadModFile(ctx context.Context) { + if len(buildList) > 0 { + return + } + + Init() + if modRoot == "" { + Target = module.Version{Path: "command-line-arguments"} + targetPrefix = "command-line-arguments" + buildList = []module.Version{Target} + return + } + + gomod := ModFilePath() + data, err := lockedfile.Read(gomod) + if err != nil { + base.Fatalf("go: %v", err) + } + + var fixed bool + f, err := modfile.Parse(gomod, data, fixVersion(ctx, &fixed)) + if err != nil { + // Errors returned by modfile.Parse begin with file:line. + base.Fatalf("go: errors parsing go.mod:\n%s\n", err) + } + modFile = f + index = indexModFile(data, f, fixed) + + if f.Module == nil { + // No module declaration. Must add module path. + base.Fatalf("go: no module declaration in go.mod. To specify the module path:\n\tgo mod edit -module=example.com/mod") + } + + if err := checkModulePathLax(f.Module.Mod.Path); err != nil { + base.Fatalf("go: %v", err) + } + + setDefaultBuildMod() // possibly enable automatic vendoring + modFileToBuildList() + if cfg.BuildMod == "vendor" { + readVendorList() + checkVendorConsistency() + } +} + +// CreateModFile initializes a new module by creating a go.mod file. +// +// If modPath is empty, CreateModFile will attempt to infer the path from the +// directory location within GOPATH. +// +// If a vendoring configuration file is present, CreateModFile will attempt to +// translate it to go.mod directives. The resulting build list may not be +// exactly the same as in the legacy configuration (for example, we can't get +// packages at multiple versions from the same module). +func CreateModFile(ctx context.Context, modPath string) { + modRoot = base.Cwd + Init() + modFilePath := ModFilePath() + if _, err := fsys.Stat(modFilePath); err == nil { + base.Fatalf("go: %s already exists", modFilePath) + } + + if modPath == "" { + var err error + modPath, err = findModulePath(modRoot) + if err != nil { + base.Fatalf("go: %v", err) + } + } else if err := checkModulePathLax(modPath); err != nil { + base.Fatalf("go: %v", err) + } + + fmt.Fprintf(os.Stderr, "go: creating new go.mod: module %s\n", modPath) + modFile = new(modfile.File) + modFile.AddModuleStmt(modPath) + addGoStmt() // Add the go directive before converted module requirements. + + convertedFrom, err := convertLegacyConfig(modPath) + if convertedFrom != "" { + fmt.Fprintf(os.Stderr, "go: copying requirements from %s\n", base.ShortPath(convertedFrom)) + } + if err != nil { + base.Fatalf("go: %v", err) + } + + modFileToBuildList() + WriteGoMod() + + // Suggest running 'go mod tidy' unless the project is empty. Even if we + // imported all the correct requirements above, we're probably missing + // some sums, so the next build command in -mod=readonly will likely fail. + // + // We look for non-hidden .go files or subdirectories to determine whether + // this is an existing project. Walking the tree for packages would be more + // accurate, but could take much longer. + empty := true + files, _ := os.ReadDir(modRoot) + for _, f := range files { + name := f.Name() + if strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") { + continue + } + if strings.HasSuffix(name, ".go") || f.IsDir() { + empty = false + break + } + } + if !empty { + fmt.Fprintf(os.Stderr, "go: to add module requirements and sums:\n\tgo mod tidy\n") + } +} + +// checkModulePathLax checks that the path meets some minimum requirements +// to avoid confusing users or the module cache. The requirements are weaker +// than those of module.CheckPath to allow room for weakening module path +// requirements in the future, but strong enough to help users avoid significant +// problems. +func checkModulePathLax(p string) error { + // TODO(matloob): Replace calls of this function in this CL with calls + // to module.CheckImportPath once it's been laxened, if it becomes laxened. + // See golang.org/issue/29101 for a discussion about whether to make CheckImportPath + // more lax or more strict. + + errorf := func(format string, args ...interface{}) error { + return fmt.Errorf("invalid module path %q: %s", p, fmt.Sprintf(format, args...)) + } + + // Disallow shell characters " ' * < > ? ` | to avoid triggering bugs + // with file systems and subcommands. Disallow file path separators : and \ + // because path separators other than / will confuse the module cache. + // See fileNameOK in golang.org/x/mod/module/module.go. + shellChars := "`" + `\"'*<>?|` + fsChars := `\:` + if i := strings.IndexAny(p, shellChars); i >= 0 { + return errorf("contains disallowed shell character %q", p[i]) + } + if i := strings.IndexAny(p, fsChars); i >= 0 { + return errorf("contains disallowed path separator character %q", p[i]) + } + + // Ensure path.IsAbs and build.IsLocalImport are false, and that the path is + // invariant under path.Clean, also to avoid confusing the module cache. + if path.IsAbs(p) { + return errorf("is an absolute path") + } + if build.IsLocalImport(p) { + return errorf("is a local import path") + } + if path.Clean(p) != p { + return errorf("is not clean") + } + + return nil +} + +// fixVersion returns a modfile.VersionFixer implemented using the Query function. +// +// It resolves commit hashes and branch names to versions, +// canonicalizes versions that appeared in early vgo drafts, +// and does nothing for versions that already appear to be canonical. +// +// The VersionFixer sets 'fixed' if it ever returns a non-canonical version. +func fixVersion(ctx context.Context, fixed *bool) modfile.VersionFixer { + return func(path, vers string) (resolved string, err error) { + defer func() { + if err == nil && resolved != vers { + *fixed = true + } + }() + + // Special case: remove the old -gopkgin- hack. + if strings.HasPrefix(path, "gopkg.in/") && strings.Contains(vers, "-gopkgin-") { + vers = vers[strings.Index(vers, "-gopkgin-")+len("-gopkgin-"):] + } + + // fixVersion is called speculatively on every + // module, version pair from every go.mod file. + // Avoid the query if it looks OK. + _, pathMajor, ok := module.SplitPathVersion(path) + if !ok { + return "", &module.ModuleError{ + Path: path, + Err: &module.InvalidVersionError{ + Version: vers, + Err: fmt.Errorf("malformed module path %q", path), + }, + } + } + if vers != "" && module.CanonicalVersion(vers) == vers { + if err := module.CheckPathMajor(vers, pathMajor); err != nil { + return "", module.VersionError(module.Version{Path: path, Version: vers}, err) + } + return vers, nil + } + + info, err := Query(ctx, path, vers, "", nil) + if err != nil { + return "", err + } + return info.Version, nil + } +} + +// AllowMissingModuleImports allows import paths to be resolved to modules +// when there is no module root. Normally, this is forbidden because it's slow +// and there's no way to make the result reproducible, but some commands +// like 'go get' are expected to do this. +func AllowMissingModuleImports() { + allowMissingModuleImports = true +} + +// modFileToBuildList initializes buildList from the modFile. +func modFileToBuildList() { + Target = modFile.Module.Mod + targetPrefix = Target.Path + if rel := search.InDir(base.Cwd, cfg.GOROOTsrc); rel != "" { + targetInGorootSrc = true + if Target.Path == "std" { + targetPrefix = "" + } + } + + list := []module.Version{Target} + for _, r := range modFile.Require { + if index != nil && index.exclude[r.Mod] { + if cfg.BuildMod == "mod" { + fmt.Fprintf(os.Stderr, "go: dropping requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + } else { + fmt.Fprintf(os.Stderr, "go: ignoring requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + } + } else { + list = append(list, r.Mod) + } + } + buildList = list +} + +// setDefaultBuildMod sets a default value for cfg.BuildMod if the -mod flag +// wasn't provided. setDefaultBuildMod may be called multiple times. +func setDefaultBuildMod() { + if cfg.BuildModExplicit { + // Don't override an explicit '-mod=' argument. + return + } + + if cfg.CmdName == "get" || strings.HasPrefix(cfg.CmdName, "mod ") { + // 'get' and 'go mod' commands may update go.mod automatically. + // TODO(jayconrod): should this narrower? Should 'go mod download' or + // 'go mod graph' update go.mod by default? + cfg.BuildMod = "mod" + return + } + if modRoot == "" { + cfg.BuildMod = "readonly" + return + } + + if fi, err := fsys.Stat(filepath.Join(modRoot, "vendor")); err == nil && fi.IsDir() { + modGo := "unspecified" + if index != nil && index.goVersionV != "" { + if semver.Compare(index.goVersionV, "v1.14") >= 0 { + // The Go version is at least 1.14, and a vendor directory exists. + // Set -mod=vendor by default. + cfg.BuildMod = "vendor" + cfg.BuildModReason = "Go version in go.mod is at least 1.14 and vendor directory exists." + return + } else { + modGo = index.goVersionV[1:] + } + } + + // Since a vendor directory exists, we should record why we didn't use it. + // This message won't normally be shown, but it may appear with import errors. + cfg.BuildModReason = fmt.Sprintf("Go version in go.mod is %s, so vendor directory was not used.", modGo) + } + + cfg.BuildMod = "readonly" +} + +// convertLegacyConfig imports module requirements from a legacy vendoring +// configuration file, if one is present. +func convertLegacyConfig(modPath string) (from string, err error) { + for _, name := range altConfigs { + cfg := filepath.Join(modRoot, name) + data, err := os.ReadFile(cfg) + if err == nil { + convert := modconv.Converters[name] + if convert == nil { + return "", nil + } + cfg = filepath.ToSlash(cfg) + err := modconv.ConvertLegacyConfig(modFile, cfg, data) + return name, err + } + } + return "", nil +} + +// addGoStmt adds a go directive to the go.mod file if it does not already include one. +// The 'go' version added, if any, is the latest version supported by this toolchain. +func addGoStmt() { + if modFile.Go != nil && modFile.Go.Version != "" { + return + } + tags := build.Default.ReleaseTags + version := tags[len(tags)-1] + if !strings.HasPrefix(version, "go") || !modfile.GoVersionRE.MatchString(version[2:]) { + base.Fatalf("go: unrecognized default version %q", version) + } + if err := modFile.AddGoStmt(version[2:]); err != nil { + base.Fatalf("go: internal error: %v", err) + } +} + +var altConfigs = []string{ + "Gopkg.lock", + + "GLOCKFILE", + "Godeps/Godeps.json", + "dependencies.tsv", + "glide.lock", + "vendor.conf", + "vendor.yml", + "vendor/manifest", + "vendor/vendor.json", + + ".git/config", +} + +func findModuleRoot(dir string) (root string) { + if dir == "" { + panic("dir not set") + } + dir = filepath.Clean(dir) + + // Look for enclosing go.mod. + for { + if fi, err := fsys.Stat(filepath.Join(dir, "go.mod")); err == nil && !fi.IsDir() { + return dir + } + d := filepath.Dir(dir) + if d == dir { + break + } + dir = d + } + return "" +} + +func findAltConfig(dir string) (root, name string) { + if dir == "" { + panic("dir not set") + } + dir = filepath.Clean(dir) + if rel := search.InDir(dir, cfg.BuildContext.GOROOT); rel != "" { + // Don't suggest creating a module from $GOROOT/.git/config + // or a config file found in any parent of $GOROOT (see #34191). + return "", "" + } + for { + for _, name := range altConfigs { + if fi, err := fsys.Stat(filepath.Join(dir, name)); err == nil && !fi.IsDir() { + return dir, name + } + } + d := filepath.Dir(dir) + if d == dir { + break + } + dir = d + } + return "", "" +} + +func findModulePath(dir string) (string, error) { + // TODO(bcmills): once we have located a plausible module path, we should + // query version control (if available) to verify that it matches the major + // version of the most recent tag. + // See https://golang.org/issue/29433, https://golang.org/issue/27009, and + // https://golang.org/issue/31549. + + // Cast about for import comments, + // first in top-level directory, then in subdirectories. + list, _ := os.ReadDir(dir) + for _, info := range list { + if info.Type().IsRegular() && strings.HasSuffix(info.Name(), ".go") { + if com := findImportComment(filepath.Join(dir, info.Name())); com != "" { + return com, nil + } + } + } + for _, info1 := range list { + if info1.IsDir() { + files, _ := os.ReadDir(filepath.Join(dir, info1.Name())) + for _, info2 := range files { + if info2.Type().IsRegular() && strings.HasSuffix(info2.Name(), ".go") { + if com := findImportComment(filepath.Join(dir, info1.Name(), info2.Name())); com != "" { + return path.Dir(com), nil + } + } + } + } + } + + // Look for Godeps.json declaring import path. + data, _ := os.ReadFile(filepath.Join(dir, "Godeps/Godeps.json")) + var cfg1 struct{ ImportPath string } + json.Unmarshal(data, &cfg1) + if cfg1.ImportPath != "" { + return cfg1.ImportPath, nil + } + + // Look for vendor.json declaring import path. + data, _ = os.ReadFile(filepath.Join(dir, "vendor/vendor.json")) + var cfg2 struct{ RootPath string } + json.Unmarshal(data, &cfg2) + if cfg2.RootPath != "" { + return cfg2.RootPath, nil + } + + // Look for path in GOPATH. + var badPathErr error + for _, gpdir := range filepath.SplitList(cfg.BuildContext.GOPATH) { + if gpdir == "" { + continue + } + if rel := search.InDir(dir, filepath.Join(gpdir, "src")); rel != "" && rel != "." { + path := filepath.ToSlash(rel) + // TODO(matloob): replace this with module.CheckImportPath + // once it's been laxened. + // Only checkModulePathLax here. There are some unpublishable + // module names that are compatible with checkModulePathLax + // but they already work in GOPATH so don't break users + // trying to do a build with modules. gorelease will alert users + // publishing their modules to fix their paths. + if err := checkModulePathLax(path); err != nil { + badPathErr = err + break + } + return path, nil + } + } + + reason := "outside GOPATH, module path must be specified" + if badPathErr != nil { + // return a different error message if the module was in GOPATH, but + // the module path determined above would be an invalid path. + reason = fmt.Sprintf("bad module path inferred from directory in GOPATH: %v", badPathErr) + } + msg := `cannot determine module path for source directory %s (%s) + +Example usage: + 'go mod init example.com/m' to initialize a v0 or v1 module + 'go mod init example.com/m/v2' to initialize a v2 module + +Run 'go help mod init' for more information. +` + return "", fmt.Errorf(msg, dir, reason) +} + +var ( + importCommentRE = lazyregexp.New(`(?m)^package[ \t]+[^ \t\r\n/]+[ \t]+//[ \t]+import[ \t]+(\"[^"]+\")[ \t]*\r?\n`) +) + +func findImportComment(file string) string { + data, err := os.ReadFile(file) + if err != nil { + return "" + } + m := importCommentRE.FindSubmatch(data) + if m == nil { + return "" + } + path, err := strconv.Unquote(string(m[1])) + if err != nil { + return "" + } + return path +} + +var allowWriteGoMod = true + +// DisallowWriteGoMod causes future calls to WriteGoMod to do nothing at all. +func DisallowWriteGoMod() { + allowWriteGoMod = false +} + +// AllowWriteGoMod undoes the effect of DisallowWriteGoMod: +// future calls to WriteGoMod will update go.mod if needed. +// Note that any past calls have been discarded, so typically +// a call to AlowWriteGoMod should be followed by a call to WriteGoMod. +func AllowWriteGoMod() { + allowWriteGoMod = true +} + +// MinReqs returns a Reqs with minimal additional dependencies of Target, +// as will be written to go.mod. +func MinReqs() mvs.Reqs { + retain := append([]string{}, additionalExplicitRequirements...) + for _, m := range buildList[1:] { + _, explicit := index.require[m] + if explicit || loaded.direct[m.Path] { + retain = append(retain, m.Path) + } + } + sort.Strings(retain) + str.Uniq(&retain) + min, err := mvs.Req(Target, retain, &mvsReqs{buildList: buildList}) + if err != nil { + base.Fatalf("go: %v", err) + } + return &mvsReqs{buildList: append([]module.Version{Target}, min...)} +} + +// WriteGoMod writes the current build list back to go.mod. +func WriteGoMod() { + // If we're using -mod=vendor we basically ignored + // go.mod, so definitely don't try to write back our + // incomplete view of the world. + if !allowWriteGoMod || cfg.BuildMod == "vendor" { + return + } + + // If we aren't in a module, we don't have anywhere to write a go.mod file. + if modRoot == "" { + return + } + + if cfg.BuildMod != "readonly" { + addGoStmt() + } + + if loaded != nil { + reqs := MinReqs() + min, err := reqs.Required(Target) + if err != nil { + base.Fatalf("go: %v", err) + } + var list []*modfile.Require + for _, m := range min { + list = append(list, &modfile.Require{ + Mod: m, + Indirect: !loaded.direct[m.Path], + }) + } + modFile.SetRequire(list) + } + modFile.Cleanup() + + dirty := index.modFileIsDirty(modFile) + if dirty && cfg.BuildMod == "readonly" { + // If we're about to fail due to -mod=readonly, + // prefer to report a dirty go.mod over a dirty go.sum + if cfg.BuildModExplicit { + base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly") + } else if cfg.BuildModReason != "" { + base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly\n\t(%s)", cfg.BuildModReason) + } else { + base.Fatalf("go: updates to go.mod needed; to update it:\n\tgo mod tidy") + } + } + + if !dirty && cfg.CmdName != "mod tidy" { + // The go.mod file has the same semantic content that it had before + // (but not necessarily the same exact bytes). + // Don't write go.mod, but write go.sum in case we added or trimmed sums. + // 'go mod init' shouldn't write go.sum, since it will be incomplete. + if cfg.CmdName != "mod init" { + modfetch.WriteGoSum(keepSums(true)) + } + return + } + + new, err := modFile.Format() + if err != nil { + base.Fatalf("go: %v", err) + } + defer func() { + // At this point we have determined to make the go.mod file on disk equal to new. + index = indexModFile(new, modFile, false) + + // Update go.sum after releasing the side lock and refreshing the index. + // 'go mod init' shouldn't write go.sum, since it will be incomplete. + if cfg.CmdName != "mod init" { + modfetch.WriteGoSum(keepSums(true)) + } + }() + + // Make a best-effort attempt to acquire the side lock, only to exclude + // previous versions of the 'go' command from making simultaneous edits. + if unlock, err := modfetch.SideLock(); err == nil { + defer unlock() + } + + errNoChange := errors.New("no update needed") + + err = lockedfile.Transform(ModFilePath(), func(old []byte) ([]byte, error) { + if bytes.Equal(old, new) { + // The go.mod file is already equal to new, possibly as the result of some + // other process. + return nil, errNoChange + } + + if index != nil && !bytes.Equal(old, index.data) { + // The contents of the go.mod file have changed. In theory we could add all + // of the new modules to the build list, recompute, and check whether any + // module in *our* build list got bumped to a different version, but that's + // a lot of work for marginal benefit. Instead, fail the command: if users + // want to run concurrent commands, they need to start with a complete, + // consistent module definition. + return nil, fmt.Errorf("existing contents have changed since last read") + } + + return new, nil + }) + + if err != nil && err != errNoChange { + base.Fatalf("go: updating go.mod: %v", err) + } +} + +// keepSums returns a set of module sums to preserve in go.sum. The set +// includes entries for all modules used to load packages (according to +// the last load function such as LoadPackages or ImportFromFiles). +// It also contains entries for go.mod files needed for MVS (the version +// of these entries ends with "/go.mod"). +// +// If keepBuildListZips is true, the set also includes sums for zip files for +// all modules in the build list with replacements applied. 'go get' and +// 'go mod download' may add sums to this set when adding a requirement on a +// module without a root package or when downloading a direct or indirect +// dependency. +func keepSums(keepBuildListZips bool) map[module.Version]bool { + // Re-derive the build list using the current list of direct requirements. + // Keep the sum for the go.mod of each visited module version (or its + // replacement). + modkey := func(m module.Version) module.Version { + return module.Version{Path: m.Path, Version: m.Version + "/go.mod"} + } + keep := make(map[module.Version]bool) + var mu sync.Mutex + reqs := &keepSumReqs{ + Reqs: &mvsReqs{buildList: buildList}, + visit: func(m module.Version) { + // If we build using a replacement module, keep the sum for the replacement, + // since that's the code we'll actually use during a build. + mu.Lock() + r := Replacement(m) + if r.Path == "" { + keep[modkey(m)] = true + } else { + keep[modkey(r)] = true + } + mu.Unlock() + }, + } + buildList, err := mvs.BuildList(Target, reqs) + if err != nil { + panic(fmt.Sprintf("unexpected error reloading build list: %v", err)) + } + + actualMods := make(map[string]module.Version) + for _, m := range buildList[1:] { + if r := Replacement(m); r.Path != "" { + actualMods[m.Path] = r + } else { + actualMods[m.Path] = m + } + } + + // Add entries for modules in the build list with paths that are prefixes of + // paths of loaded packages. We need to retain sums for modules needed to + // report ambiguous import errors. We use our re-derived build list, + // since the global build list may have been tidied. + if loaded != nil { + for _, pkg := range loaded.pkgs { + if pkg.testOf != nil || pkg.inStd || module.CheckImportPath(pkg.path) != nil { + continue + } + for prefix := pkg.path; prefix != "."; prefix = path.Dir(prefix) { + if m, ok := actualMods[prefix]; ok { + keep[m] = true + } + } + } + } + + // Add entries for the zip of each module in the build list. + // We might not need all of these (tidy does not add them), but they may be + // added by a specific 'go get' or 'go mod download' command to resolve + // missing import sum errors. + if keepBuildListZips { + for _, m := range actualMods { + keep[m] = true + } + } + + return keep +} + +// keepSumReqs embeds another Reqs implementation. The Required method +// calls visit for each version in the module graph. +type keepSumReqs struct { + mvs.Reqs + visit func(module.Version) +} + +func (r *keepSumReqs) Required(m module.Version) ([]module.Version, error) { + r.visit(m) + return r.Reqs.Required(m) +} + +func TrimGoSum() { + // Don't retain sums for the zip file of every module in the build list. + // We may not need them all to build the main module's packages. + keepBuildListZips := false + modfetch.TrimGoSum(keepSums(keepBuildListZips)) +} diff --git a/src/cmd/go/internal/modload/list.go b/src/cmd/go/internal/modload/list.go new file mode 100644 index 0000000..7b1aa7f --- /dev/null +++ b/src/cmd/go/internal/modload/list.go @@ -0,0 +1,191 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "context" + "errors" + "fmt" + "os" + "runtime" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/modinfo" + "cmd/go/internal/search" + + "golang.org/x/mod/module" +) + +func ListModules(ctx context.Context, args []string, listU, listVersions, listRetracted bool) []*modinfo.ModulePublic { + mods := listModules(ctx, args, listVersions, listRetracted) + + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + if listU || listVersions || listRetracted { + for _, m := range mods { + add := func(m *modinfo.ModulePublic) { + sem <- token{} + go func() { + if listU { + addUpdate(ctx, m) + } + if listVersions { + addVersions(ctx, m, listRetracted) + } + if listRetracted || listU { + addRetraction(ctx, m) + } + <-sem + }() + } + + add(m) + if m.Replace != nil { + add(m.Replace) + } + } + } + // Fill semaphore channel to wait for all tasks to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} + } + + return mods +} + +func listModules(ctx context.Context, args []string, listVersions, listRetracted bool) []*modinfo.ModulePublic { + LoadAllModules(ctx) + if len(args) == 0 { + return []*modinfo.ModulePublic{moduleInfo(ctx, buildList[0], true, listRetracted)} + } + + var mods []*modinfo.ModulePublic + matchedBuildList := make([]bool, len(buildList)) + for _, arg := range args { + if strings.Contains(arg, `\`) { + base.Fatalf("go: module paths never use backslash") + } + if search.IsRelativePath(arg) { + base.Fatalf("go: cannot use relative path %s to specify module", arg) + } + if !HasModRoot() && (arg == "all" || strings.Contains(arg, "...")) { + base.Fatalf("go: cannot match %q: %v", arg, ErrNoModRoot) + } + if i := strings.Index(arg, "@"); i >= 0 { + path := arg[:i] + vers := arg[i+1:] + var current string + for _, m := range buildList { + if m.Path == path { + current = m.Version + break + } + } + + allowed := CheckAllowed + if IsRevisionQuery(vers) || listRetracted { + // Allow excluded and retracted versions if the user asked for a + // specific revision or used 'go list -retracted'. + allowed = nil + } + info, err := Query(ctx, path, vers, current, allowed) + if err != nil { + mods = append(mods, &modinfo.ModulePublic{ + Path: path, + Version: vers, + Error: modinfoError(path, vers, err), + }) + continue + } + mod := moduleInfo(ctx, module.Version{Path: path, Version: info.Version}, false, listRetracted) + mods = append(mods, mod) + continue + } + + // Module path or pattern. + var match func(string) bool + var literal bool + if arg == "all" { + match = func(string) bool { return true } + } else if strings.Contains(arg, "...") { + match = search.MatchPattern(arg) + } else { + match = func(p string) bool { return arg == p } + literal = true + } + matched := false + for i, m := range buildList { + if i == 0 && !HasModRoot() { + // The root module doesn't actually exist: omit it. + continue + } + if match(m.Path) { + matched = true + if !matchedBuildList[i] { + matchedBuildList[i] = true + mods = append(mods, moduleInfo(ctx, m, true, listRetracted)) + } + } + } + if !matched { + if literal { + if listVersions { + // Don't make the user provide an explicit '@latest' when they're + // explicitly asking what the available versions are. + // Instead, resolve the module, even if it isn't an existing dependency. + info, err := Query(ctx, arg, "latest", "", nil) + if err == nil { + mod := moduleInfo(ctx, module.Version{Path: arg, Version: info.Version}, false, listRetracted) + mods = append(mods, mod) + } else { + mods = append(mods, &modinfo.ModulePublic{ + Path: arg, + Error: modinfoError(arg, "", err), + }) + } + continue + } + if cfg.BuildMod == "vendor" { + // In vendor mode, we can't determine whether a missing module is “a + // known dependency” because the module graph is incomplete. + // Give a more explicit error message. + mods = append(mods, &modinfo.ModulePublic{ + Path: arg, + Error: modinfoError(arg, "", errors.New("can't resolve module using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)")), + }) + } else { + mods = append(mods, &modinfo.ModulePublic{ + Path: arg, + Error: modinfoError(arg, "", errors.New("not a known dependency")), + }) + } + } else { + fmt.Fprintf(os.Stderr, "warning: pattern %q matched no module dependencies\n", arg) + } + } + } + + return mods +} + +// modinfoError wraps an error to create an error message in +// modinfo.ModuleError with minimal redundancy. +func modinfoError(path, vers string, err error) *modinfo.ModuleError { + var nerr *NoMatchingVersionError + var merr *module.ModuleError + if errors.As(err, &nerr) { + // NoMatchingVersionError contains the query, so we don't mention the + // query again in ModuleError. + err = &module.ModuleError{Path: path, Err: err} + } else if !errors.As(err, &merr) { + // If the error does not contain path and version, wrap it in a + // module.ModuleError. + err = &module.ModuleError{Path: path, Version: vers, Err: err} + } + + return &modinfo.ModuleError{Err: err.Error()} +} diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go new file mode 100644 index 0000000..154fc3c --- /dev/null +++ b/src/cmd/go/internal/modload/load.go @@ -0,0 +1,1364 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +// This file contains the module-mode package loader, as well as some accessory +// functions pertaining to the package import graph. +// +// There are two exported entry points into package loading — LoadPackages and +// ImportFromFiles — both implemented in terms of loadFromRoots, which itself +// manipulates an instance of the loader struct. +// +// Although most of the loading state is maintained in the loader struct, +// one key piece - the build list - is a global, so that it can be modified +// separate from the loading operation, such as during "go get" +// upgrades/downgrades or in "go mod" operations. +// TODO(#40775): It might be nice to make the loader take and return +// a buildList rather than hard-coding use of the global. +// +// Loading is an iterative process. On each iteration, we try to load the +// requested packages and their transitive imports, then try to resolve modules +// for any imported packages that are still missing. +// +// The first step of each iteration identifies a set of “root” packages. +// Normally the root packages are exactly those matching the named pattern +// arguments. However, for the "all" meta-pattern, the final set of packages is +// computed from the package import graph, and therefore cannot be an initial +// input to loading that graph. Instead, the root packages for the "all" pattern +// are those contained in the main module, and allPatternIsRoot parameter to the +// loader instructs it to dynamically expand those roots to the full "all" +// pattern as loading progresses. +// +// The pkgInAll flag on each loadPkg instance tracks whether that +// package is known to match the "all" meta-pattern. +// A package matches the "all" pattern if: +// - it is in the main module, or +// - it is imported by any test in the main module, or +// - it is imported by another package in "all", or +// - the main module specifies a go version ≤ 1.15, and the package is imported +// by a *test of* another package in "all". +// +// When we implement lazy loading, we will record the modules providing packages +// in "all" even when we are only loading individual packages, so we set the +// pkgInAll flag regardless of the whether the "all" pattern is a root. +// (This is necessary to maintain the “import invariant” described in +// https://golang.org/design/36460-lazy-module-loading.) +// +// Because "go mod vendor" prunes out the tests of vendored packages, the +// behavior of the "all" pattern with -mod=vendor in Go 1.11–1.15 is the same +// as the "all" pattern (regardless of the -mod flag) in 1.16+. +// The allClosesOverTests parameter to the loader indicates whether the "all" +// pattern should close over tests (as in Go 1.11–1.15) or stop at only those +// packages transitively imported by the packages and tests in the main module +// ("all" in Go 1.16+ and "go mod vendor" in Go 1.11+). +// +// Note that it is possible for a loaded package NOT to be in "all" even when we +// are loading the "all" pattern. For example, packages that are transitive +// dependencies of other roots named on the command line must be loaded, but are +// not in "all". (The mod_notall test illustrates this behavior.) +// Similarly, if the LoadTests flag is set but the "all" pattern does not close +// over test dependencies, then when we load the test of a package that is in +// "all" but outside the main module, the dependencies of that test will not +// necessarily themselves be in "all". (That configuration does not arise in Go +// 1.11–1.15, but it will be possible in Go 1.16+.) +// +// Loading proceeds from the roots, using a parallel work-queue with a limit on +// the amount of active work (to avoid saturating disks, CPU cores, and/or +// network connections). Each package is added to the queue the first time it is +// imported by another package. When we have finished identifying the imports of +// a package, we add the test for that package if it is needed. A test may be +// needed if: +// - the package matches a root pattern and tests of the roots were requested, or +// - the package is in the main module and the "all" pattern is requested +// (because the "all" pattern includes the dependencies of tests in the main +// module), or +// - the package is in "all" and the definition of "all" we are using includes +// dependencies of tests (as is the case in Go ≤1.15). +// +// After all available packages have been loaded, we examine the results to +// identify any requested or imported packages that are still missing, and if +// so, which modules we could add to the module graph in order to make the +// missing packages available. We add those to the module graph and iterate, +// until either all packages resolve successfully or we cannot identify any +// module that would resolve any remaining missing package. +// +// If the main module is “tidy” (that is, if "go mod tidy" is a no-op for it) +// and all requested packages are in "all", then loading completes in a single +// iteration. +// TODO(bcmills): We should also be able to load in a single iteration if the +// requested packages all come from modules that are themselves tidy, regardless +// of whether those packages are in "all". Today, that requires two iterations +// if those packages are not found in existing dependencies of the main module. + +import ( + "bytes" + "context" + "errors" + "fmt" + "go/build" + "io/fs" + "os" + "path" + pathpkg "path" + "path/filepath" + "reflect" + "runtime" + "sort" + "strings" + "sync" + "sync/atomic" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/imports" + "cmd/go/internal/modfetch" + "cmd/go/internal/mvs" + "cmd/go/internal/par" + "cmd/go/internal/search" + "cmd/go/internal/str" + + "golang.org/x/mod/module" +) + +// loaded is the most recently-used package loader. +// It holds details about individual packages. +var loaded *loader + +// PackageOpts control the behavior of the LoadPackages function. +type PackageOpts struct { + // Tags are the build tags in effect (as interpreted by the + // cmd/go/internal/imports package). + // If nil, treated as equivalent to imports.Tags(). + Tags map[string]bool + + // ResolveMissingImports indicates that we should attempt to add module + // dependencies as needed to resolve imports of packages that are not found. + // + // For commands that support the -mod flag, resolving imports may still fail + // if the flag is set to "readonly" (the default) or "vendor". + ResolveMissingImports bool + + // AllowPackage, if non-nil, is called after identifying the module providing + // each package. If AllowPackage returns a non-nil error, that error is set + // for the package, and the imports and test of that package will not be + // loaded. + // + // AllowPackage may be invoked concurrently by multiple goroutines, + // and may be invoked multiple times for a given package path. + AllowPackage func(ctx context.Context, path string, mod module.Version) error + + // LoadTests loads the test dependencies of each package matching a requested + // pattern. If ResolveMissingImports is also true, test dependencies will be + // resolved if missing. + LoadTests bool + + // UseVendorAll causes the "all" package pattern to be interpreted as if + // running "go mod vendor" (or building with "-mod=vendor"). + // + // This is a no-op for modules that declare 'go 1.16' or higher, for which this + // is the default (and only) interpretation of the "all" pattern in module mode. + UseVendorAll bool + + // AllowErrors indicates that LoadPackages should not terminate the process if + // an error occurs. + AllowErrors bool + + // SilenceErrors indicates that LoadPackages should not print errors + // that occur while loading packages. SilenceErrors implies AllowErrors. + SilenceErrors bool + + // SilenceMissingStdImports indicates that LoadPackages should not print + // errors or terminate the process if an imported package is missing, and the + // import path looks like it might be in the standard library (perhaps in a + // future version). + SilenceMissingStdImports bool + + // SilenceUnmatchedWarnings suppresses the warnings normally emitted for + // patterns that did not match any packages. + SilenceUnmatchedWarnings bool +} + +// LoadPackages identifies the set of packages matching the given patterns and +// loads the packages in the import graph rooted at that set. +func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (matches []*search.Match, loadedPackages []string) { + LoadModFile(ctx) + if opts.Tags == nil { + opts.Tags = imports.Tags() + } + + patterns = search.CleanPatterns(patterns) + matches = make([]*search.Match, 0, len(patterns)) + allPatternIsRoot := false + for _, pattern := range patterns { + matches = append(matches, search.NewMatch(pattern)) + if pattern == "all" { + allPatternIsRoot = true + } + } + + updateMatches := func(ld *loader) { + for _, m := range matches { + switch { + case m.IsLocal(): + // Evaluate list of file system directories on first iteration. + if m.Dirs == nil { + matchLocalDirs(m) + } + + // Make a copy of the directory list and translate to import paths. + // Note that whether a directory corresponds to an import path + // changes as the build list is updated, and a directory can change + // from not being in the build list to being in it and back as + // the exact version of a particular module increases during + // the loader iterations. + m.Pkgs = m.Pkgs[:0] + for _, dir := range m.Dirs { + pkg, err := resolveLocalPackage(dir) + if err != nil { + if !m.IsLiteral() && (err == errPkgIsBuiltin || err == errPkgIsGorootSrc) { + continue // Don't include "builtin" or GOROOT/src in wildcard patterns. + } + + // If we're outside of a module, ensure that the failure mode + // indicates that. + ModRoot() + + if ld != nil { + m.AddError(err) + } + continue + } + m.Pkgs = append(m.Pkgs, pkg) + } + + case m.IsLiteral(): + m.Pkgs = []string{m.Pattern()} + + case strings.Contains(m.Pattern(), "..."): + m.Errs = m.Errs[:0] + matchPackages(ctx, m, opts.Tags, includeStd, buildList) + + case m.Pattern() == "all": + if ld == nil { + // The initial roots are the packages in the main module. + // loadFromRoots will expand that to "all". + m.Errs = m.Errs[:0] + matchPackages(ctx, m, opts.Tags, omitStd, []module.Version{Target}) + } else { + // Starting with the packages in the main module, + // enumerate the full list of "all". + m.Pkgs = ld.computePatternAll() + } + + case m.Pattern() == "std" || m.Pattern() == "cmd": + if m.Pkgs == nil { + m.MatchPackages() // Locate the packages within GOROOT/src. + } + + default: + panic(fmt.Sprintf("internal error: modload missing case for pattern %s", m.Pattern())) + } + } + } + + loaded = loadFromRoots(loaderParams{ + PackageOpts: opts, + + allClosesOverTests: index.allPatternClosesOverTests() && !opts.UseVendorAll, + allPatternIsRoot: allPatternIsRoot, + + listRoots: func() (roots []string) { + updateMatches(nil) + for _, m := range matches { + roots = append(roots, m.Pkgs...) + } + return roots + }, + }) + + // One last pass to finalize wildcards. + updateMatches(loaded) + + // Report errors, if any. + checkMultiplePaths() + for _, pkg := range loaded.pkgs { + if pkg.err != nil { + if sumErr := (*ImportMissingSumError)(nil); errors.As(pkg.err, &sumErr) { + if importer := pkg.stack; importer != nil { + sumErr.importer = importer.path + sumErr.importerVersion = importer.mod.Version + sumErr.importerIsTest = importer.testOf != nil + } + } + silence := opts.SilenceErrors + if stdErr := (*ImportMissingError)(nil); errors.As(pkg.err, &stdErr) && + stdErr.isStd && opts.SilenceMissingStdImports { + silence = true + } + + if !silence { + if opts.AllowErrors { + fmt.Fprintf(os.Stderr, "%s: %v\n", pkg.stackText(), pkg.err) + } else { + base.Errorf("%s: %v", pkg.stackText(), pkg.err) + } + } + } + if !pkg.isTest() { + loadedPackages = append(loadedPackages, pkg.path) + } + } + if !opts.SilenceErrors { + // Also list errors in matching patterns (such as directory permission + // errors for wildcard patterns). + for _, match := range matches { + for _, err := range match.Errs { + if opts.AllowErrors { + fmt.Fprintf(os.Stderr, "%v\n", err) + } else { + base.Errorf("%v", err) + } + } + } + } + base.ExitIfErrors() + + if !opts.SilenceUnmatchedWarnings { + search.WarnUnmatched(matches) + } + + // Success! Update go.mod (if needed) and return the results. + WriteGoMod() + sort.Strings(loadedPackages) + return matches, loadedPackages +} + +// matchLocalDirs is like m.MatchDirs, but tries to avoid scanning directories +// outside of the standard library and active modules. +func matchLocalDirs(m *search.Match) { + if !m.IsLocal() { + panic(fmt.Sprintf("internal error: resolveLocalDirs on non-local pattern %s", m.Pattern())) + } + + if i := strings.Index(m.Pattern(), "..."); i >= 0 { + // The pattern is local, but it is a wildcard. Its packages will + // only resolve to paths if they are inside of the standard + // library, the main module, or some dependency of the main + // module. Verify that before we walk the filesystem: a filesystem + // walk in a directory like /var or /etc can be very expensive! + dir := filepath.Dir(filepath.Clean(m.Pattern()[:i+3])) + absDir := dir + if !filepath.IsAbs(dir) { + absDir = filepath.Join(base.Cwd, dir) + } + if search.InDir(absDir, cfg.GOROOTsrc) == "" && search.InDir(absDir, ModRoot()) == "" && pathInModuleCache(absDir) == "" { + m.Dirs = []string{} + m.AddError(fmt.Errorf("directory prefix %s outside available modules", base.ShortPath(absDir))) + return + } + } + + m.MatchDirs() +} + +// resolveLocalPackage resolves a filesystem path to a package path. +func resolveLocalPackage(dir string) (string, error) { + var absDir string + if filepath.IsAbs(dir) { + absDir = filepath.Clean(dir) + } else { + absDir = filepath.Join(base.Cwd, dir) + } + + bp, err := cfg.BuildContext.ImportDir(absDir, 0) + if err != nil && (bp == nil || len(bp.IgnoredGoFiles) == 0) { + // golang.org/issue/32917: We should resolve a relative path to a + // package path only if the relative path actually contains the code + // for that package. + // + // If the named directory does not exist or contains no Go files, + // the package does not exist. + // Other errors may affect package loading, but not resolution. + if _, err := fsys.Stat(absDir); err != nil { + if os.IsNotExist(err) { + // Canonicalize OS-specific errors to errDirectoryNotFound so that error + // messages will be easier for users to search for. + return "", &fs.PathError{Op: "stat", Path: absDir, Err: errDirectoryNotFound} + } + return "", err + } + if _, noGo := err.(*build.NoGoError); noGo { + // A directory that does not contain any Go source files — even ignored + // ones! — is not a Go package, and we can't resolve it to a package + // path because that path could plausibly be provided by some other + // module. + // + // Any other error indicates that the package “exists” (at least in the + // sense that it cannot exist in any other module), but has some other + // problem (such as a syntax error). + return "", err + } + } + + if modRoot != "" && absDir == modRoot { + if absDir == cfg.GOROOTsrc { + return "", errPkgIsGorootSrc + } + return targetPrefix, nil + } + + // Note: The checks for @ here are just to avoid misinterpreting + // the module cache directories (formerly GOPATH/src/mod/foo@v1.5.2/bar). + // It's not strictly necessary but helpful to keep the checks. + if modRoot != "" && strings.HasPrefix(absDir, modRoot+string(filepath.Separator)) && !strings.Contains(absDir[len(modRoot):], "@") { + suffix := filepath.ToSlash(absDir[len(modRoot):]) + if strings.HasPrefix(suffix, "/vendor/") { + if cfg.BuildMod != "vendor" { + return "", fmt.Errorf("without -mod=vendor, directory %s has no package path", absDir) + } + + readVendorList() + pkg := strings.TrimPrefix(suffix, "/vendor/") + if _, ok := vendorPkgModule[pkg]; !ok { + return "", fmt.Errorf("directory %s is not a package listed in vendor/modules.txt", absDir) + } + return pkg, nil + } + + if targetPrefix == "" { + pkg := strings.TrimPrefix(suffix, "/") + if pkg == "builtin" { + // "builtin" is a pseudo-package with a real source file. + // It's not included in "std", so it shouldn't resolve from "." + // within module "std" either. + return "", errPkgIsBuiltin + } + return pkg, nil + } + + pkg := targetPrefix + suffix + if _, ok, err := dirInModule(pkg, targetPrefix, modRoot, true); err != nil { + return "", err + } else if !ok { + return "", &PackageNotInModuleError{Mod: Target, Pattern: pkg} + } + return pkg, nil + } + + if sub := search.InDir(absDir, cfg.GOROOTsrc); sub != "" && sub != "." && !strings.Contains(sub, "@") { + pkg := filepath.ToSlash(sub) + if pkg == "builtin" { + return "", errPkgIsBuiltin + } + return pkg, nil + } + + pkg := pathInModuleCache(absDir) + if pkg == "" { + return "", fmt.Errorf("directory %s outside available modules", base.ShortPath(absDir)) + } + return pkg, nil +} + +var ( + errDirectoryNotFound = errors.New("directory not found") + errPkgIsGorootSrc = errors.New("GOROOT/src is not an importable package") + errPkgIsBuiltin = errors.New(`"builtin" is a pseudo-package, not an importable package`) +) + +// pathInModuleCache returns the import path of the directory dir, +// if dir is in the module cache copy of a module in our build list. +func pathInModuleCache(dir string) string { + tryMod := func(m module.Version) (string, bool) { + var root string + var err error + if repl := Replacement(m); repl.Path != "" && repl.Version == "" { + root = repl.Path + if !filepath.IsAbs(root) { + root = filepath.Join(ModRoot(), root) + } + } else if repl.Path != "" { + root, err = modfetch.DownloadDir(repl) + } else { + root, err = modfetch.DownloadDir(m) + } + if err != nil { + return "", false + } + + sub := search.InDir(dir, root) + if sub == "" { + return "", false + } + sub = filepath.ToSlash(sub) + if strings.Contains(sub, "/vendor/") || strings.HasPrefix(sub, "vendor/") || strings.Contains(sub, "@") { + return "", false + } + + return path.Join(m.Path, filepath.ToSlash(sub)), true + } + + for _, m := range buildList[1:] { + if importPath, ok := tryMod(m); ok { + // checkMultiplePaths ensures that a module can be used for at most one + // requirement, so this must be it. + return importPath + } + } + return "" +} + +// ImportFromFiles adds modules to the build list as needed +// to satisfy the imports in the named Go source files. +func ImportFromFiles(ctx context.Context, gofiles []string) { + LoadModFile(ctx) + + tags := imports.Tags() + imports, testImports, err := imports.ScanFiles(gofiles, tags) + if err != nil { + base.Fatalf("go: %v", err) + } + + loaded = loadFromRoots(loaderParams{ + PackageOpts: PackageOpts{ + Tags: tags, + ResolveMissingImports: true, + }, + allClosesOverTests: index.allPatternClosesOverTests(), + listRoots: func() (roots []string) { + roots = append(roots, imports...) + roots = append(roots, testImports...) + return roots + }, + }) + WriteGoMod() +} + +// DirImportPath returns the effective import path for dir, +// provided it is within the main module, or else returns ".". +func DirImportPath(dir string) string { + if !HasModRoot() { + return "." + } + LoadModFile(context.TODO()) + + if !filepath.IsAbs(dir) { + dir = filepath.Join(base.Cwd, dir) + } else { + dir = filepath.Clean(dir) + } + + if dir == modRoot { + return targetPrefix + } + if strings.HasPrefix(dir, modRoot+string(filepath.Separator)) { + suffix := filepath.ToSlash(dir[len(modRoot):]) + if strings.HasPrefix(suffix, "/vendor/") { + return strings.TrimPrefix(suffix, "/vendor/") + } + return targetPrefix + suffix + } + return "." +} + +// TargetPackages returns the list of packages in the target (top-level) module +// matching pattern, which may be relative to the working directory, under all +// build tag settings. +func TargetPackages(ctx context.Context, pattern string) *search.Match { + // TargetPackages is relative to the main module, so ensure that the main + // module is a thing that can contain packages. + LoadModFile(ctx) + ModRoot() + + m := search.NewMatch(pattern) + matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{Target}) + return m +} + +// ImportMap returns the actual package import path +// for an import path found in source code. +// If the given import path does not appear in the source code +// for the packages that have been loaded, ImportMap returns the empty string. +func ImportMap(path string) string { + pkg, ok := loaded.pkgCache.Get(path).(*loadPkg) + if !ok { + return "" + } + return pkg.path +} + +// PackageDir returns the directory containing the source code +// for the package named by the import path. +func PackageDir(path string) string { + pkg, ok := loaded.pkgCache.Get(path).(*loadPkg) + if !ok { + return "" + } + return pkg.dir +} + +// PackageModule returns the module providing the package named by the import path. +func PackageModule(path string) module.Version { + pkg, ok := loaded.pkgCache.Get(path).(*loadPkg) + if !ok { + return module.Version{} + } + return pkg.mod +} + +// PackageImports returns the imports for the package named by the import path. +// Test imports will be returned as well if tests were loaded for the package +// (i.e., if "all" was loaded or if LoadTests was set and the path was matched +// by a command line argument). PackageImports will return nil for +// unknown package paths. +func PackageImports(path string) (imports, testImports []string) { + pkg, ok := loaded.pkgCache.Get(path).(*loadPkg) + if !ok { + return nil, nil + } + imports = make([]string, len(pkg.imports)) + for i, p := range pkg.imports { + imports[i] = p.path + } + if pkg.test != nil { + testImports = make([]string, len(pkg.test.imports)) + for i, p := range pkg.test.imports { + testImports[i] = p.path + } + } + return imports, testImports +} + +// Lookup returns the source directory, import path, and any loading error for +// the package at path as imported from the package in parentDir. +// Lookup requires that one of the Load functions in this package has already +// been called. +func Lookup(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) { + if path == "" { + panic("Lookup called with empty package path") + } + + if parentIsStd { + path = loaded.stdVendor(parentPath, path) + } + pkg, ok := loaded.pkgCache.Get(path).(*loadPkg) + if !ok { + // The loader should have found all the relevant paths. + // There are a few exceptions, though: + // - during go list without -test, the p.Resolve calls to process p.TestImports and p.XTestImports + // end up here to canonicalize the import paths. + // - during any load, non-loaded packages like "unsafe" end up here. + // - during any load, build-injected dependencies like "runtime/cgo" end up here. + // - because we ignore appengine/* in the module loader, + // the dependencies of any actual appengine/* library end up here. + dir := findStandardImportPath(path) + if dir != "" { + return dir, path, nil + } + return "", "", errMissing + } + return pkg.dir, pkg.path, pkg.err +} + +// A loader manages the process of loading information about +// the required packages for a particular build, +// checking that the packages are available in the module set, +// and updating the module set if needed. +type loader struct { + loaderParams + + work *par.Queue + + // reset on each iteration + roots []*loadPkg + pkgCache *par.Cache // package path (string) → *loadPkg + pkgs []*loadPkg // transitive closure of loaded packages and tests; populated in buildStacks + + // computed at end of iterations + direct map[string]bool // imported directly by main module +} + +// loaderParams configure the packages loaded by, and the properties reported +// by, a loader instance. +type loaderParams struct { + PackageOpts + + allClosesOverTests bool // Does the "all" pattern include the transitive closure of tests of packages in "all"? + allPatternIsRoot bool // Is the "all" pattern an additional root? + + listRoots func() []string +} + +func (ld *loader) reset() { + select { + case <-ld.work.Idle(): + default: + panic("loader.reset when not idle") + } + + ld.roots = nil + ld.pkgCache = new(par.Cache) + ld.pkgs = nil +} + +// A loadPkg records information about a single loaded package. +type loadPkg struct { + // Populated at construction time: + path string // import path + testOf *loadPkg + + // Populated at construction time and updated by (*loader).applyPkgFlags: + flags atomicLoadPkgFlags + + // Populated by (*loader).load: + mod module.Version // module providing package + dir string // directory containing source code + err error // error loading package + imports []*loadPkg // packages imported by this one + testImports []string // test-only imports, saved for use by pkg.test. + inStd bool + + // Populated by (*loader).pkgTest: + testOnce sync.Once + test *loadPkg + + // Populated by postprocessing in (*loader).buildStacks: + stack *loadPkg // package importing this one in minimal import stack for this pkg +} + +// loadPkgFlags is a set of flags tracking metadata about a package. +type loadPkgFlags int8 + +const ( + // pkgInAll indicates that the package is in the "all" package pattern, + // regardless of whether we are loading the "all" package pattern. + // + // When the pkgInAll flag and pkgImportsLoaded flags are both set, the caller + // who set the last of those flags must propagate the pkgInAll marking to all + // of the imports of the marked package. + // + // A test is marked with pkgInAll if that test would promote the packages it + // imports to be in "all" (such as when the test is itself within the main + // module, or when ld.allClosesOverTests is true). + pkgInAll loadPkgFlags = 1 << iota + + // pkgIsRoot indicates that the package matches one of the root package + // patterns requested by the caller. + // + // If LoadTests is set, then when pkgIsRoot and pkgImportsLoaded are both set, + // the caller who set the last of those flags must populate a test for the + // package (in the pkg.test field). + // + // If the "all" pattern is included as a root, then non-test packages in "all" + // are also roots (and must be marked pkgIsRoot). + pkgIsRoot + + // pkgImportsLoaded indicates that the imports and testImports fields of a + // loadPkg have been populated. + pkgImportsLoaded +) + +// has reports whether all of the flags in cond are set in f. +func (f loadPkgFlags) has(cond loadPkgFlags) bool { + return f&cond == cond +} + +// An atomicLoadPkgFlags stores a loadPkgFlags for which individual flags can be +// added atomically. +type atomicLoadPkgFlags struct { + bits int32 +} + +// update sets the given flags in af (in addition to any flags already set). +// +// update returns the previous flag state so that the caller may determine which +// flags were newly-set. +func (af *atomicLoadPkgFlags) update(flags loadPkgFlags) (old loadPkgFlags) { + for { + old := atomic.LoadInt32(&af.bits) + new := old | int32(flags) + if new == old || atomic.CompareAndSwapInt32(&af.bits, old, new) { + return loadPkgFlags(old) + } + } +} + +// has reports whether all of the flags in cond are set in af. +func (af *atomicLoadPkgFlags) has(cond loadPkgFlags) bool { + return loadPkgFlags(atomic.LoadInt32(&af.bits))&cond == cond +} + +// isTest reports whether pkg is a test of another package. +func (pkg *loadPkg) isTest() bool { + return pkg.testOf != nil +} + +var errMissing = errors.New("cannot find package") + +// loadFromRoots attempts to load the build graph needed to process a set of +// root packages and their dependencies. +// +// The set of root packages is returned by the params.listRoots function, and +// expanded to the full set of packages by tracing imports (and possibly tests) +// as needed. +func loadFromRoots(params loaderParams) *loader { + ld := &loader{ + loaderParams: params, + work: par.NewQueue(runtime.GOMAXPROCS(0)), + } + + var err error + reqs := &mvsReqs{buildList: buildList} + buildList, err = mvs.BuildList(Target, reqs) + if err != nil { + base.Fatalf("go: %v", err) + } + + addedModuleFor := make(map[string]bool) + for { + ld.reset() + + // Load the root packages and their imports. + // Note: the returned roots can change on each iteration, + // since the expansion of package patterns depends on the + // build list we're using. + inRoots := map[*loadPkg]bool{} + for _, path := range ld.listRoots() { + root := ld.pkg(path, pkgIsRoot) + if !inRoots[root] { + ld.roots = append(ld.roots, root) + inRoots[root] = true + } + } + + // ld.pkg adds imported packages to the work queue and calls applyPkgFlags, + // which adds tests (and test dependencies) as needed. + // + // When all of the work in the queue has completed, we'll know that the + // transitive closure of dependencies has been loaded. + <-ld.work.Idle() + + ld.buildStacks() + + if !ld.ResolveMissingImports || (!HasModRoot() && !allowMissingModuleImports) { + // We've loaded as much as we can without resolving missing imports. + break + } + modAddedBy := ld.resolveMissingImports(addedModuleFor) + if len(modAddedBy) == 0 { + break + } + + // Recompute buildList with all our additions. + reqs = &mvsReqs{buildList: buildList} + buildList, err = mvs.BuildList(Target, reqs) + if err != nil { + // If an error was found in a newly added module, report the package + // import stack instead of the module requirement stack. Packages + // are more descriptive. + if err, ok := err.(*mvs.BuildListError); ok { + if pkg := modAddedBy[err.Module()]; pkg != nil { + base.Fatalf("go: %s: %v", pkg.stackText(), err.Err) + } + } + base.Fatalf("go: %v", err) + } + } + base.ExitIfErrors() + + // Compute directly referenced dependency modules. + ld.direct = make(map[string]bool) + for _, pkg := range ld.pkgs { + if pkg.mod == Target { + for _, dep := range pkg.imports { + if dep.mod.Path != "" && dep.mod.Path != Target.Path && index != nil { + _, explicit := index.require[dep.mod] + if allowWriteGoMod && cfg.BuildMod == "readonly" && !explicit { + // TODO(#40775): attach error to package instead of using + // base.Errorf. Ideally, 'go list' should not fail because of this, + // but today, LoadPackages calls WriteGoMod unconditionally, which + // would fail with a less clear message. + base.Errorf("go: %[1]s: package %[2]s imported from implicitly required module; to add missing requirements, run:\n\tgo get %[2]s@%[3]s", pkg.path, dep.path, dep.mod.Version) + } + ld.direct[dep.mod.Path] = true + } + } + } + } + base.ExitIfErrors() + + // If we didn't scan all of the imports from the main module, or didn't use + // imports.AnyTags, then we didn't necessarily load every package that + // contributes “direct” imports — so we can't safely mark existing + // dependencies as indirect-only. + // Conservatively mark those dependencies as direct. + if modFile != nil && (!ld.allPatternIsRoot || !reflect.DeepEqual(ld.Tags, imports.AnyTags())) { + for _, r := range modFile.Require { + if !r.Indirect { + ld.direct[r.Mod.Path] = true + } + } + } + + return ld +} + +// resolveMissingImports adds module dependencies to the global build list +// in order to resolve missing packages from pkgs. +// +// The newly-resolved packages are added to the addedModuleFor map, and +// resolveMissingImports returns a map from each newly-added module version to +// the first package for which that module was added. +func (ld *loader) resolveMissingImports(addedModuleFor map[string]bool) (modAddedBy map[module.Version]*loadPkg) { + var needPkgs []*loadPkg + for _, pkg := range ld.pkgs { + if pkg.err == nil { + continue + } + if pkg.isTest() { + // If we are missing a test, we are also missing its non-test version, and + // we should only add the missing import once. + continue + } + if !errors.As(pkg.err, new(*ImportMissingError)) { + // Leave other errors for Import or load.Packages to report. + continue + } + + needPkgs = append(needPkgs, pkg) + + pkg := pkg + ld.work.Add(func() { + pkg.mod, pkg.err = queryImport(context.TODO(), pkg.path) + }) + } + <-ld.work.Idle() + + modAddedBy = map[module.Version]*loadPkg{} + for _, pkg := range needPkgs { + if pkg.err != nil { + continue + } + + fmt.Fprintf(os.Stderr, "go: found %s in %s %s\n", pkg.path, pkg.mod.Path, pkg.mod.Version) + if addedModuleFor[pkg.path] { + // TODO(bcmills): This should only be an error if pkg.mod is the same + // version we already tried to add previously. + base.Fatalf("go: %s: looping trying to add package", pkg.stackText()) + } + if modAddedBy[pkg.mod] == nil { + modAddedBy[pkg.mod] = pkg + buildList = append(buildList, pkg.mod) + } + } + + return modAddedBy +} + +// pkg locates the *loadPkg for path, creating and queuing it for loading if +// needed, and updates its state to reflect the given flags. +// +// The imports of the returned *loadPkg will be loaded asynchronously in the +// ld.work queue, and its test (if requested) will also be populated once +// imports have been resolved. When ld.work goes idle, all transitive imports of +// the requested package (and its test, if requested) will have been loaded. +func (ld *loader) pkg(path string, flags loadPkgFlags) *loadPkg { + if flags.has(pkgImportsLoaded) { + panic("internal error: (*loader).pkg called with pkgImportsLoaded flag set") + } + + pkg := ld.pkgCache.Do(path, func() interface{} { + pkg := &loadPkg{ + path: path, + } + ld.applyPkgFlags(pkg, flags) + + ld.work.Add(func() { ld.load(pkg) }) + return pkg + }).(*loadPkg) + + ld.applyPkgFlags(pkg, flags) + return pkg +} + +// applyPkgFlags updates pkg.flags to set the given flags and propagate the +// (transitive) effects of those flags, possibly loading or enqueueing further +// packages as a result. +func (ld *loader) applyPkgFlags(pkg *loadPkg, flags loadPkgFlags) { + if flags == 0 { + return + } + + if flags.has(pkgInAll) && ld.allPatternIsRoot && !pkg.isTest() { + // This package matches a root pattern by virtue of being in "all". + flags |= pkgIsRoot + } + + old := pkg.flags.update(flags) + new := old | flags + if new == old || !new.has(pkgImportsLoaded) { + // We either didn't change the state of pkg, or we don't know anything about + // its dependencies yet. Either way, we can't usefully load its test or + // update its dependencies. + return + } + + if !pkg.isTest() { + // Check whether we should add (or update the flags for) a test for pkg. + // ld.pkgTest is idempotent and extra invocations are inexpensive, + // so it's ok if we call it more than is strictly necessary. + wantTest := false + switch { + case ld.allPatternIsRoot && pkg.mod == Target: + // We are loading the "all" pattern, which includes packages imported by + // tests in the main module. This package is in the main module, so we + // need to identify the imports of its test even if LoadTests is not set. + // + // (We will filter out the extra tests explicitly in computePatternAll.) + wantTest = true + + case ld.allPatternIsRoot && ld.allClosesOverTests && new.has(pkgInAll): + // This variant of the "all" pattern includes imports of tests of every + // package that is itself in "all", and pkg is in "all", so its test is + // also in "all" (as above). + wantTest = true + + case ld.LoadTests && new.has(pkgIsRoot): + // LoadTest explicitly requests tests of “the root packages”. + wantTest = true + } + + if wantTest { + var testFlags loadPkgFlags + if pkg.mod == Target || (ld.allClosesOverTests && new.has(pkgInAll)) { + // Tests of packages in the main module are in "all", in the sense that + // they cause the packages they import to also be in "all". So are tests + // of packages in "all" if "all" closes over test dependencies. + testFlags |= pkgInAll + } + ld.pkgTest(pkg, testFlags) + } + } + + if new.has(pkgInAll) && !old.has(pkgInAll|pkgImportsLoaded) { + // We have just marked pkg with pkgInAll, or we have just loaded its + // imports, or both. Now is the time to propagate pkgInAll to the imports. + for _, dep := range pkg.imports { + ld.applyPkgFlags(dep, pkgInAll) + } + } +} + +// load loads an individual package. +func (ld *loader) load(pkg *loadPkg) { + if strings.Contains(pkg.path, "@") { + // Leave for error during load. + return + } + if build.IsLocalImport(pkg.path) || filepath.IsAbs(pkg.path) { + // Leave for error during load. + // (Module mode does not allow local imports.) + return + } + + if search.IsMetaPackage(pkg.path) { + pkg.err = &invalidImportError{ + importPath: pkg.path, + err: fmt.Errorf("%q is not an importable package; see 'go help packages'", pkg.path), + } + return + } + + pkg.mod, pkg.dir, pkg.err = importFromBuildList(context.TODO(), pkg.path, buildList) + if pkg.dir == "" { + return + } + if pkg.mod == Target { + // Go ahead and mark pkg as in "all". This provides the invariant that a + // package that is *only* imported by other packages in "all" is always + // marked as such before loading its imports. + // + // We don't actually rely on that invariant at the moment, but it may + // improve efficiency somewhat and makes the behavior a bit easier to reason + // about (by reducing churn on the flag bits of dependencies), and costs + // essentially nothing (these atomic flag ops are essentially free compared + // to scanning source code for imports). + ld.applyPkgFlags(pkg, pkgInAll) + } + if ld.AllowPackage != nil { + if err := ld.AllowPackage(context.TODO(), pkg.path, pkg.mod); err != nil { + pkg.err = err + } + } + + pkg.inStd = (search.IsStandardImportPath(pkg.path) && search.InDir(pkg.dir, cfg.GOROOTsrc) != "") + + var imports, testImports []string + + if cfg.BuildContext.Compiler == "gccgo" && pkg.inStd { + // We can't scan standard packages for gccgo. + } else { + var err error + imports, testImports, err = scanDir(pkg.dir, ld.Tags) + if err != nil { + pkg.err = err + return + } + } + + pkg.imports = make([]*loadPkg, 0, len(imports)) + var importFlags loadPkgFlags + if pkg.flags.has(pkgInAll) { + importFlags = pkgInAll + } + for _, path := range imports { + if pkg.inStd { + // Imports from packages in "std" and "cmd" should resolve using + // GOROOT/src/vendor even when "std" is not the main module. + path = ld.stdVendor(pkg.path, path) + } + pkg.imports = append(pkg.imports, ld.pkg(path, importFlags)) + } + pkg.testImports = testImports + + ld.applyPkgFlags(pkg, pkgImportsLoaded) +} + +// pkgTest locates the test of pkg, creating it if needed, and updates its state +// to reflect the given flags. +// +// pkgTest requires that the imports of pkg have already been loaded (flagged +// with pkgImportsLoaded). +func (ld *loader) pkgTest(pkg *loadPkg, testFlags loadPkgFlags) *loadPkg { + if pkg.isTest() { + panic("pkgTest called on a test package") + } + + createdTest := false + pkg.testOnce.Do(func() { + pkg.test = &loadPkg{ + path: pkg.path, + testOf: pkg, + mod: pkg.mod, + dir: pkg.dir, + err: pkg.err, + inStd: pkg.inStd, + } + ld.applyPkgFlags(pkg.test, testFlags) + createdTest = true + }) + + test := pkg.test + if createdTest { + test.imports = make([]*loadPkg, 0, len(pkg.testImports)) + var importFlags loadPkgFlags + if test.flags.has(pkgInAll) { + importFlags = pkgInAll + } + for _, path := range pkg.testImports { + if pkg.inStd { + path = ld.stdVendor(test.path, path) + } + test.imports = append(test.imports, ld.pkg(path, importFlags)) + } + pkg.testImports = nil + ld.applyPkgFlags(test, pkgImportsLoaded) + } else { + ld.applyPkgFlags(test, testFlags) + } + + return test +} + +// stdVendor returns the canonical import path for the package with the given +// path when imported from the standard-library package at parentPath. +func (ld *loader) stdVendor(parentPath, path string) string { + if search.IsStandardImportPath(path) { + return path + } + + if str.HasPathPrefix(parentPath, "cmd") { + if Target.Path != "cmd" { + vendorPath := pathpkg.Join("cmd", "vendor", path) + if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { + return vendorPath + } + } + } else if Target.Path != "std" || str.HasPathPrefix(parentPath, "vendor") { + // If we are outside of the 'std' module, resolve imports from within 'std' + // to the vendor directory. + // + // Do the same for importers beginning with the prefix 'vendor/' even if we + // are *inside* of the 'std' module: the 'vendor/' packages that resolve + // globally from GOROOT/src/vendor (and are listed as part of 'go list std') + // are distinct from the real module dependencies, and cannot import + // internal packages from the real module. + // + // (Note that although the 'vendor/' packages match the 'std' *package* + // pattern, they are not part of the std *module*, and do not affect + // 'go mod tidy' and similar module commands when working within std.) + vendorPath := pathpkg.Join("vendor", path) + if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { + return vendorPath + } + } + + // Not vendored: resolve from modules. + return path +} + +// computePatternAll returns the list of packages matching pattern "all", +// starting with a list of the import paths for the packages in the main module. +func (ld *loader) computePatternAll() (all []string) { + for _, pkg := range ld.pkgs { + if pkg.flags.has(pkgInAll) && !pkg.isTest() { + all = append(all, pkg.path) + } + } + sort.Strings(all) + return all +} + +// scanDir is like imports.ScanDir but elides known magic imports from the list, +// so that we do not go looking for packages that don't really exist. +// +// The standard magic import is "C", for cgo. +// +// The only other known magic imports are appengine and appengine/*. +// These are so old that they predate "go get" and did not use URL-like paths. +// Most code today now uses google.golang.org/appengine instead, +// but not all code has been so updated. When we mostly ignore build tags +// during "go vendor", we look into "// +build appengine" files and +// may see these legacy imports. We drop them so that the module +// search does not look for modules to try to satisfy them. +func scanDir(dir string, tags map[string]bool) (imports_, testImports []string, err error) { + imports_, testImports, err = imports.ScanDir(dir, tags) + + filter := func(x []string) []string { + w := 0 + for _, pkg := range x { + if pkg != "C" && pkg != "appengine" && !strings.HasPrefix(pkg, "appengine/") && + pkg != "appengine_internal" && !strings.HasPrefix(pkg, "appengine_internal/") { + x[w] = pkg + w++ + } + } + return x[:w] + } + + return filter(imports_), filter(testImports), err +} + +// buildStacks computes minimal import stacks for each package, +// for use in error messages. When it completes, packages that +// are part of the original root set have pkg.stack == nil, +// and other packages have pkg.stack pointing at the next +// package up the import stack in their minimal chain. +// As a side effect, buildStacks also constructs ld.pkgs, +// the list of all packages loaded. +func (ld *loader) buildStacks() { + if len(ld.pkgs) > 0 { + panic("buildStacks") + } + for _, pkg := range ld.roots { + pkg.stack = pkg // sentinel to avoid processing in next loop + ld.pkgs = append(ld.pkgs, pkg) + } + for i := 0; i < len(ld.pkgs); i++ { // not range: appending to ld.pkgs in loop + pkg := ld.pkgs[i] + for _, next := range pkg.imports { + if next.stack == nil { + next.stack = pkg + ld.pkgs = append(ld.pkgs, next) + } + } + if next := pkg.test; next != nil && next.stack == nil { + next.stack = pkg + ld.pkgs = append(ld.pkgs, next) + } + } + for _, pkg := range ld.roots { + pkg.stack = nil + } +} + +// stackText builds the import stack text to use when +// reporting an error in pkg. It has the general form +// +// root imports +// other imports +// other2 tested by +// other2.test imports +// pkg +// +func (pkg *loadPkg) stackText() string { + var stack []*loadPkg + for p := pkg; p != nil; p = p.stack { + stack = append(stack, p) + } + + var buf bytes.Buffer + for i := len(stack) - 1; i >= 0; i-- { + p := stack[i] + fmt.Fprint(&buf, p.path) + if p.testOf != nil { + fmt.Fprint(&buf, ".test") + } + if i > 0 { + if stack[i-1].testOf == p { + fmt.Fprint(&buf, " tested by\n\t") + } else { + fmt.Fprint(&buf, " imports\n\t") + } + } + } + return buf.String() +} + +// why returns the text to use in "go mod why" output about the given package. +// It is less ornate than the stackText but contains the same information. +func (pkg *loadPkg) why() string { + var buf strings.Builder + var stack []*loadPkg + for p := pkg; p != nil; p = p.stack { + stack = append(stack, p) + } + + for i := len(stack) - 1; i >= 0; i-- { + p := stack[i] + if p.testOf != nil { + fmt.Fprintf(&buf, "%s.test\n", p.testOf.path) + } else { + fmt.Fprintf(&buf, "%s\n", p.path) + } + } + return buf.String() +} + +// Why returns the "go mod why" output stanza for the given package, +// without the leading # comment. +// The package graph must have been loaded already, usually by LoadPackages. +// If there is no reason for the package to be in the current build, +// Why returns an empty string. +func Why(path string) string { + pkg, ok := loaded.pkgCache.Get(path).(*loadPkg) + if !ok { + return "" + } + return pkg.why() +} + +// WhyDepth returns the number of steps in the Why listing. +// If there is no reason for the package to be in the current build, +// WhyDepth returns 0. +func WhyDepth(path string) int { + n := 0 + pkg, _ := loaded.pkgCache.Get(path).(*loadPkg) + for p := pkg; p != nil; p = p.stack { + n++ + } + return n +} diff --git a/src/cmd/go/internal/modload/modfile.go b/src/cmd/go/internal/modload/modfile.go new file mode 100644 index 0000000..c6667d0 --- /dev/null +++ b/src/cmd/go/internal/modload/modfile.go @@ -0,0 +1,591 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "strings" + "sync" + "unicode" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/lockedfile" + "cmd/go/internal/modfetch" + "cmd/go/internal/par" + "cmd/go/internal/trace" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +// narrowAllVersionV is the Go version (plus leading "v") at which the +// module-module "all" pattern no longer closes over the dependencies of +// tests outside of the main module. +const narrowAllVersionV = "v1.16" +const go116EnableNarrowAll = true + +var modFile *modfile.File + +// A modFileIndex is an index of data corresponding to a modFile +// at a specific point in time. +type modFileIndex struct { + data []byte + dataNeedsFix bool // true if fixVersion applied a change while parsing data + module module.Version + goVersionV string // GoVersion with "v" prefix + require map[module.Version]requireMeta + replace map[module.Version]module.Version + highestReplaced map[string]string // highest replaced version of each module path; empty string for wildcard-only replacements + exclude map[module.Version]bool +} + +// index is the index of the go.mod file as of when it was last read or written. +var index *modFileIndex + +type requireMeta struct { + indirect bool +} + +// CheckAllowed returns an error equivalent to ErrDisallowed if m is excluded by +// the main module's go.mod or retracted by its author. Most version queries use +// this to filter out versions that should not be used. +func CheckAllowed(ctx context.Context, m module.Version) error { + if err := CheckExclusions(ctx, m); err != nil { + return err + } + if err := CheckRetractions(ctx, m); err != nil { + return err + } + return nil +} + +// ErrDisallowed is returned by version predicates passed to Query and similar +// functions to indicate that a version should not be considered. +var ErrDisallowed = errors.New("disallowed module version") + +// CheckExclusions returns an error equivalent to ErrDisallowed if module m is +// excluded by the main module's go.mod file. +func CheckExclusions(ctx context.Context, m module.Version) error { + if index != nil && index.exclude[m] { + return module.VersionError(m, errExcluded) + } + return nil +} + +var errExcluded = &excludedError{} + +type excludedError struct{} + +func (e *excludedError) Error() string { return "excluded by go.mod" } +func (e *excludedError) Is(err error) bool { return err == ErrDisallowed } + +// CheckRetractions returns an error if module m has been retracted by +// its author. +func CheckRetractions(ctx context.Context, m module.Version) error { + if m.Version == "" { + // Main module, standard library, or file replacement module. + // Cannot be retracted. + return nil + } + + // Look up retraction information from the latest available version of + // the module. Cache retraction information so we don't parse the go.mod + // file repeatedly. + type entry struct { + retract []retraction + err error + } + path := m.Path + e := retractCache.Do(path, func() (v interface{}) { + ctx, span := trace.StartSpan(ctx, "checkRetractions "+path) + defer span.Done() + + if repl := Replacement(module.Version{Path: m.Path}); repl.Path != "" { + // All versions of the module were replaced with a local directory. + // Don't load retractions. + return &entry{nil, nil} + } + + // Find the latest version of the module. + // Ignore exclusions from the main module's go.mod. + const ignoreSelected = "" + var allowAll AllowedFunc + rev, err := Query(ctx, path, "latest", ignoreSelected, allowAll) + if err != nil { + return &entry{nil, err} + } + + // Load go.mod for that version. + // If the version is replaced, we'll load retractions from the replacement. + // + // If there's an error loading the go.mod, we'll return it here. + // These errors should generally be ignored by callers of checkRetractions, + // since they happen frequently when we're offline. These errors are not + // equivalent to ErrDisallowed, so they may be distinguished from + // retraction errors. + // + // We load the raw file here: the go.mod file may have a different module + // path that we expect if the module or its repository was renamed. + // We still want to apply retractions to other aliases of the module. + rm := module.Version{Path: path, Version: rev.Version} + if repl := Replacement(rm); repl.Path != "" { + rm = repl + } + summary, err := rawGoModSummary(rm) + if err != nil { + return &entry{nil, err} + } + return &entry{summary.retract, nil} + }).(*entry) + + if err := e.err; err != nil { + // Attribute the error to the version being checked, not the version from + // which the retractions were to be loaded. + var mErr *module.ModuleError + if errors.As(err, &mErr) { + err = mErr.Err + } + return &retractionLoadingError{m: m, err: err} + } + + var rationale []string + isRetracted := false + for _, r := range e.retract { + if semver.Compare(r.Low, m.Version) <= 0 && semver.Compare(m.Version, r.High) <= 0 { + isRetracted = true + if r.Rationale != "" { + rationale = append(rationale, r.Rationale) + } + } + } + if isRetracted { + return module.VersionError(m, &ModuleRetractedError{Rationale: rationale}) + } + return nil +} + +var retractCache par.Cache + +type ModuleRetractedError struct { + Rationale []string +} + +func (e *ModuleRetractedError) Error() string { + msg := "retracted by module author" + if len(e.Rationale) > 0 { + // This is meant to be a short error printed on a terminal, so just + // print the first rationale. + msg += ": " + ShortRetractionRationale(e.Rationale[0]) + } + return msg +} + +func (e *ModuleRetractedError) Is(err error) bool { + return err == ErrDisallowed +} + +type retractionLoadingError struct { + m module.Version + err error +} + +func (e *retractionLoadingError) Error() string { + return fmt.Sprintf("loading module retractions for %v: %v", e.m, e.err) +} + +func (e *retractionLoadingError) Unwrap() error { + return e.err +} + +// ShortRetractionRationale returns a retraction rationale string that is safe +// to print in a terminal. It returns hard-coded strings if the rationale +// is empty, too long, or contains non-printable characters. +func ShortRetractionRationale(rationale string) string { + const maxRationaleBytes = 500 + if i := strings.Index(rationale, "\n"); i >= 0 { + rationale = rationale[:i] + } + rationale = strings.TrimSpace(rationale) + if rationale == "" { + return "retracted by module author" + } + if len(rationale) > maxRationaleBytes { + return "(rationale omitted: too long)" + } + for _, r := range rationale { + if !unicode.IsGraphic(r) && !unicode.IsSpace(r) { + return "(rationale omitted: contains non-printable characters)" + } + } + // NOTE: the go.mod parser rejects invalid UTF-8, so we don't check that here. + return rationale +} + +// Replacement returns the replacement for mod, if any, from go.mod. +// If there is no replacement for mod, Replacement returns +// a module.Version with Path == "". +func Replacement(mod module.Version) module.Version { + if index != nil { + if r, ok := index.replace[mod]; ok { + return r + } + if r, ok := index.replace[module.Version{Path: mod.Path}]; ok { + return r + } + } + return module.Version{} +} + +// indexModFile rebuilds the index of modFile. +// If modFile has been changed since it was first read, +// modFile.Cleanup must be called before indexModFile. +func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileIndex { + i := new(modFileIndex) + i.data = data + i.dataNeedsFix = needsFix + + i.module = module.Version{} + if modFile.Module != nil { + i.module = modFile.Module.Mod + } + + i.goVersionV = "" + if modFile.Go != nil { + // We're going to use the semver package to compare Go versions, so go ahead + // and add the "v" prefix it expects once instead of every time. + i.goVersionV = "v" + modFile.Go.Version + } + + i.require = make(map[module.Version]requireMeta, len(modFile.Require)) + for _, r := range modFile.Require { + i.require[r.Mod] = requireMeta{indirect: r.Indirect} + } + + i.replace = make(map[module.Version]module.Version, len(modFile.Replace)) + for _, r := range modFile.Replace { + if prev, dup := i.replace[r.Old]; dup && prev != r.New { + base.Fatalf("go: conflicting replacements for %v:\n\t%v\n\t%v", r.Old, prev, r.New) + } + i.replace[r.Old] = r.New + } + + i.highestReplaced = make(map[string]string) + for _, r := range modFile.Replace { + v, ok := i.highestReplaced[r.Old.Path] + if !ok || semver.Compare(r.Old.Version, v) > 0 { + i.highestReplaced[r.Old.Path] = r.Old.Version + } + } + + i.exclude = make(map[module.Version]bool, len(modFile.Exclude)) + for _, x := range modFile.Exclude { + i.exclude[x.Mod] = true + } + + return i +} + +// allPatternClosesOverTests reports whether the "all" pattern includes +// dependencies of tests outside the main module (as in Go 1.11–1.15). +// (Otherwise — as in Go 1.16+ — the "all" pattern includes only the packages +// transitively *imported by* the packages and tests in the main module.) +func (i *modFileIndex) allPatternClosesOverTests() bool { + if !go116EnableNarrowAll { + return true + } + if i != nil && semver.Compare(i.goVersionV, narrowAllVersionV) < 0 { + // The module explicitly predates the change in "all" for lazy loading, so + // continue to use the older interpretation. (If i == nil, we not in any + // module at all and should use the latest semantics.) + return true + } + return false +} + +// modFileIsDirty reports whether the go.mod file differs meaningfully +// from what was indexed. +// If modFile has been changed (even cosmetically) since it was first read, +// modFile.Cleanup must be called before modFileIsDirty. +func (i *modFileIndex) modFileIsDirty(modFile *modfile.File) bool { + if i == nil { + return modFile != nil + } + + if i.dataNeedsFix { + return true + } + + if modFile.Module == nil { + if i.module != (module.Version{}) { + return true + } + } else if modFile.Module.Mod != i.module { + return true + } + + if modFile.Go == nil { + if i.goVersionV != "" { + return true + } + } else if "v"+modFile.Go.Version != i.goVersionV { + if i.goVersionV == "" && cfg.BuildMod == "readonly" { + // go.mod files did not always require a 'go' version, so do not error out + // if one is missing — we may be inside an older module in the module + // cache, and should bias toward providing useful behavior. + } else { + return true + } + } + + if len(modFile.Require) != len(i.require) || + len(modFile.Replace) != len(i.replace) || + len(modFile.Exclude) != len(i.exclude) { + return true + } + + for _, r := range modFile.Require { + if meta, ok := i.require[r.Mod]; !ok { + return true + } else if r.Indirect != meta.indirect { + if cfg.BuildMod == "readonly" { + // The module's requirements are consistent; only the "// indirect" + // comments that are wrong. But those are only guaranteed to be accurate + // after a "go mod tidy" — it's a good idea to run those before + // committing a change, but it's certainly not mandatory. + } else { + return true + } + } + } + + for _, r := range modFile.Replace { + if r.New != i.replace[r.Old] { + return true + } + } + + for _, x := range modFile.Exclude { + if !i.exclude[x.Mod] { + return true + } + } + + return false +} + +// rawGoVersion records the Go version parsed from each module's go.mod file. +// +// If a module is replaced, the version of the replacement is keyed by the +// replacement module.Version, not the version being replaced. +var rawGoVersion sync.Map // map[module.Version]string + +// A modFileSummary is a summary of a go.mod file for which we do not need to +// retain complete information — for example, the go.mod file of a dependency +// module. +type modFileSummary struct { + module module.Version + goVersionV string // GoVersion with "v" prefix + require []module.Version + retract []retraction +} + +// A retraction consists of a retracted version interval and rationale. +// retraction is like modfile.Retract, but it doesn't point to the syntax tree. +type retraction struct { + modfile.VersionInterval + Rationale string +} + +// goModSummary returns a summary of the go.mod file for module m, +// taking into account any replacements for m, exclusions of its dependencies, +// and/or vendoring. +// +// m must be a version in the module graph, reachable from the Target module. +// In readonly mode, the go.sum file must contain an entry for m's go.mod file +// (or its replacement). goModSummary must not be called for the Target module +// itself, as its requirements may change. Use rawGoModSummary for other +// module versions. +// +// The caller must not modify the returned summary. +func goModSummary(m module.Version) (*modFileSummary, error) { + if m == Target { + panic("internal error: goModSummary called on the Target module") + } + + if cfg.BuildMod == "vendor" { + summary := &modFileSummary{ + module: module.Version{Path: m.Path}, + } + if vendorVersion[m.Path] != m.Version { + // This module is not vendored, so packages cannot be loaded from it and + // it cannot be relevant to the build. + return summary, nil + } + + // For every module other than the target, + // return the full list of modules from modules.txt. + readVendorList() + + // TODO(#36876): Load the "go" version from vendor/modules.txt and store it + // in rawGoVersion with the appropriate key. + + // We don't know what versions the vendored module actually relies on, + // so assume that it requires everything. + summary.require = vendorList + return summary, nil + } + + actual := Replacement(m) + if actual.Path == "" { + actual = m + } + if HasModRoot() && cfg.BuildMod == "readonly" && actual.Version != "" { + key := module.Version{Path: actual.Path, Version: actual.Version + "/go.mod"} + if !modfetch.HaveSum(key) { + suggestion := fmt.Sprintf("; to add it:\n\tgo mod download %s", m.Path) + return nil, module.VersionError(actual, &sumMissingError{suggestion: suggestion}) + } + } + summary, err := rawGoModSummary(actual) + if err != nil { + return nil, err + } + + if actual.Version == "" { + // The actual module is a filesystem-local replacement, for which we have + // unfortunately not enforced any sort of invariants about module lines or + // matching module paths. Anything goes. + // + // TODO(bcmills): Remove this special-case, update tests, and add a + // release note. + } else { + if summary.module.Path == "" { + return nil, module.VersionError(actual, errors.New("parsing go.mod: missing module line")) + } + + // In theory we should only allow mpath to be unequal to m.Path here if the + // version that we fetched lacks an explicit go.mod file: if the go.mod file + // is explicit, then it should match exactly (to ensure that imports of other + // packages within the module are interpreted correctly). Unfortunately, we + // can't determine that information from the module proxy protocol: we'll have + // to leave that validation for when we load actual packages from within the + // module. + if mpath := summary.module.Path; mpath != m.Path && mpath != actual.Path { + return nil, module.VersionError(actual, fmt.Errorf(`parsing go.mod: + module declares its path as: %s + but was required as: %s`, mpath, m.Path)) + } + } + + if index != nil && len(index.exclude) > 0 { + // Drop any requirements on excluded versions. + // Don't modify the cached summary though, since we might need the raw + // summary separately. + haveExcludedReqs := false + for _, r := range summary.require { + if index.exclude[r] { + haveExcludedReqs = true + break + } + } + if haveExcludedReqs { + s := new(modFileSummary) + *s = *summary + s.require = make([]module.Version, 0, len(summary.require)) + for _, r := range summary.require { + if !index.exclude[r] { + s.require = append(s.require, r) + } + } + summary = s + } + } + return summary, nil +} + +// rawGoModSummary returns a new summary of the go.mod file for module m, +// ignoring all replacements that may apply to m and excludes that may apply to +// its dependencies. +// +// rawGoModSummary cannot be used on the Target module. +func rawGoModSummary(m module.Version) (*modFileSummary, error) { + if m == Target { + panic("internal error: rawGoModSummary called on the Target module") + } + + type cached struct { + summary *modFileSummary + err error + } + c := rawGoModSummaryCache.Do(m, func() interface{} { + summary := new(modFileSummary) + var f *modfile.File + if m.Version == "" { + // m is a replacement module with only a file path. + dir := m.Path + if !filepath.IsAbs(dir) { + dir = filepath.Join(ModRoot(), dir) + } + gomod := filepath.Join(dir, "go.mod") + + data, err := lockedfile.Read(gomod) + if err != nil { + return cached{nil, module.VersionError(m, fmt.Errorf("reading %s: %v", base.ShortPath(gomod), err))} + } + f, err = modfile.ParseLax(gomod, data, nil) + if err != nil { + return cached{nil, module.VersionError(m, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err))} + } + } else { + if !semver.IsValid(m.Version) { + // Disallow the broader queries supported by fetch.Lookup. + base.Fatalf("go: internal error: %s@%s: unexpected invalid semantic version", m.Path, m.Version) + } + + data, err := modfetch.GoMod(m.Path, m.Version) + if err != nil { + return cached{nil, err} + } + f, err = modfile.ParseLax("go.mod", data, nil) + if err != nil { + return cached{nil, module.VersionError(m, fmt.Errorf("parsing go.mod: %v", err))} + } + } + + if f.Module != nil { + summary.module = f.Module.Mod + } + if f.Go != nil && f.Go.Version != "" { + rawGoVersion.LoadOrStore(m, f.Go.Version) + summary.goVersionV = "v" + f.Go.Version + } + if len(f.Require) > 0 { + summary.require = make([]module.Version, 0, len(f.Require)) + for _, req := range f.Require { + summary.require = append(summary.require, req.Mod) + } + } + if len(f.Retract) > 0 { + summary.retract = make([]retraction, 0, len(f.Retract)) + for _, ret := range f.Retract { + summary.retract = append(summary.retract, retraction{ + VersionInterval: ret.VersionInterval, + Rationale: ret.Rationale, + }) + } + } + + return cached{summary, nil} + }).(cached) + + return c.summary, c.err +} + +var rawGoModSummaryCache par.Cache // module.Version → rawGoModSummary result diff --git a/src/cmd/go/internal/modload/mvs.go b/src/cmd/go/internal/modload/mvs.go new file mode 100644 index 0000000..3101519 --- /dev/null +++ b/src/cmd/go/internal/modload/mvs.go @@ -0,0 +1,113 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "context" + "errors" + "os" + "sort" + + "cmd/go/internal/modfetch" + + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +// mvsReqs implements mvs.Reqs for module semantic versions, +// with any exclusions or replacements applied internally. +type mvsReqs struct { + buildList []module.Version +} + +func (r *mvsReqs) Required(mod module.Version) ([]module.Version, error) { + if mod == Target { + // Use the build list as it existed when r was constructed, not the current + // global build list. + return r.buildList[1:], nil + } + + if mod.Version == "none" { + return nil, nil + } + + summary, err := goModSummary(mod) + if err != nil { + return nil, err + } + return summary.require, nil +} + +// Max returns the maximum of v1 and v2 according to semver.Compare. +// +// As a special case, the version "" is considered higher than all other +// versions. The main module (also known as the target) has no version and must +// be chosen over other versions of the same module in the module dependency +// graph. +func (*mvsReqs) Max(v1, v2 string) string { + if v1 != "" && (v2 == "" || semver.Compare(v1, v2) == -1) { + return v2 + } + return v1 +} + +// Upgrade is a no-op, here to implement mvs.Reqs. +// The upgrade logic for go get -u is in ../modget/get.go. +func (*mvsReqs) Upgrade(m module.Version) (module.Version, error) { + return m, nil +} + +func versions(ctx context.Context, path string, allowed AllowedFunc) ([]string, error) { + // Note: modfetch.Lookup and repo.Versions are cached, + // so there's no need for us to add extra caching here. + var versions []string + err := modfetch.TryProxies(func(proxy string) error { + repo, err := lookupRepo(proxy, path) + if err != nil { + return err + } + allVersions, err := repo.Versions("") + if err != nil { + return err + } + allowedVersions := make([]string, 0, len(allVersions)) + for _, v := range allVersions { + if err := allowed(ctx, module.Version{Path: path, Version: v}); err == nil { + allowedVersions = append(allowedVersions, v) + } else if !errors.Is(err, ErrDisallowed) { + return err + } + } + versions = allowedVersions + return nil + }) + return versions, err +} + +// Previous returns the tagged version of m.Path immediately prior to +// m.Version, or version "none" if no prior version is tagged. +// +// Since the version of Target is not found in the version list, +// it has no previous version. +func (*mvsReqs) Previous(m module.Version) (module.Version, error) { + // TODO(golang.org/issue/38714): thread tracing context through MVS. + + if m == Target { + return module.Version{Path: m.Path, Version: "none"}, nil + } + + list, err := versions(context.TODO(), m.Path, CheckAllowed) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return module.Version{Path: m.Path, Version: "none"}, nil + } + return module.Version{}, err + } + i := sort.Search(len(list), func(i int) bool { return semver.Compare(list[i], m.Version) >= 0 }) + if i > 0 { + return module.Version{Path: m.Path, Version: list[i-1]}, nil + } + return module.Version{Path: m.Path, Version: "none"}, nil +} diff --git a/src/cmd/go/internal/modload/mvs_test.go b/src/cmd/go/internal/modload/mvs_test.go new file mode 100644 index 0000000..50e93c3 --- /dev/null +++ b/src/cmd/go/internal/modload/mvs_test.go @@ -0,0 +1,31 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "testing" +) + +func TestReqsMax(t *testing.T) { + type testCase struct { + a, b, want string + } + reqs := new(mvsReqs) + for _, tc := range []testCase{ + {a: "v0.1.0", b: "v0.2.0", want: "v0.2.0"}, + {a: "v0.2.0", b: "v0.1.0", want: "v0.2.0"}, + {a: "", b: "v0.1.0", want: ""}, // "" is Target.Version + {a: "v0.1.0", b: "", want: ""}, + {a: "none", b: "v0.1.0", want: "v0.1.0"}, + {a: "v0.1.0", b: "none", want: "v0.1.0"}, + {a: "none", b: "", want: ""}, + {a: "", b: "none", want: ""}, + } { + max := reqs.Max(tc.a, tc.b) + if max != tc.want { + t.Errorf("(%T).Max(%q, %q) = %q; want %q", reqs, tc.a, tc.b, max, tc.want) + } + } +} diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go new file mode 100644 index 0000000..8affd17 --- /dev/null +++ b/src/cmd/go/internal/modload/query.go @@ -0,0 +1,1111 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + pathpkg "path" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "cmd/go/internal/cfg" + "cmd/go/internal/imports" + "cmd/go/internal/modfetch" + "cmd/go/internal/search" + "cmd/go/internal/str" + "cmd/go/internal/trace" + + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +// Query looks up a revision of a given module given a version query string. +// The module must be a complete module path. +// The version must take one of the following forms: +// +// - the literal string "latest", denoting the latest available, allowed +// tagged version, with non-prereleases preferred over prereleases. +// If there are no tagged versions in the repo, latest returns the most +// recent commit. +// - the literal string "upgrade", equivalent to "latest" except that if +// current is a newer version, current will be returned (see below). +// - the literal string "patch", denoting the latest available tagged version +// with the same major and minor number as current (see below). +// - v1, denoting the latest available tagged version v1.x.x. +// - v1.2, denoting the latest available tagged version v1.2.x. +// - v1.2.3, a semantic version string denoting that tagged version. +// - <v1.2.3, <=v1.2.3, >v1.2.3, >=v1.2.3, +// denoting the version closest to the target and satisfying the given operator, +// with non-prereleases preferred over prereleases. +// - a repository commit identifier or tag, denoting that commit. +// +// current denotes the currently-selected version of the module; it may be +// "none" if no version is currently selected, or "" if the currently-selected +// version is unknown or should not be considered. If query is +// "upgrade" or "patch", current will be returned if it is a newer +// semantic version or a chronologically later pseudo-version than the +// version that would otherwise be chosen. This prevents accidental downgrades +// from newer pre-release or development versions. +// +// The allowed function (which may be nil) is used to filter out unsuitable +// versions (see AllowedFunc documentation for details). If the query refers to +// a specific revision (for example, "master"; see IsRevisionQuery), and the +// revision is disallowed by allowed, Query returns the error. If the query +// does not refer to a specific revision (for example, "latest"), Query +// acts as if versions disallowed by allowed do not exist. +// +// If path is the path of the main module and the query is "latest", +// Query returns Target.Version as the version. +func Query(ctx context.Context, path, query, current string, allowed AllowedFunc) (*modfetch.RevInfo, error) { + var info *modfetch.RevInfo + err := modfetch.TryProxies(func(proxy string) (err error) { + info, err = queryProxy(ctx, proxy, path, query, current, allowed) + return err + }) + return info, err +} + +// AllowedFunc is used by Query and other functions to filter out unsuitable +// versions, for example, those listed in exclude directives in the main +// module's go.mod file. +// +// An AllowedFunc returns an error equivalent to ErrDisallowed for an unsuitable +// version. Any other error indicates the function was unable to determine +// whether the version should be allowed, for example, the function was unable +// to fetch or parse a go.mod file containing retractions. Typically, errors +// other than ErrDisallowd may be ignored. +type AllowedFunc func(context.Context, module.Version) error + +var errQueryDisabled error = queryDisabledError{} + +type queryDisabledError struct{} + +func (queryDisabledError) Error() string { + if cfg.BuildModReason == "" { + return fmt.Sprintf("cannot query module due to -mod=%s", cfg.BuildMod) + } + return fmt.Sprintf("cannot query module due to -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) +} + +func queryProxy(ctx context.Context, proxy, path, query, current string, allowed AllowedFunc) (*modfetch.RevInfo, error) { + ctx, span := trace.StartSpan(ctx, "modload.queryProxy "+path+" "+query) + defer span.Done() + + if current != "" && current != "none" && !semver.IsValid(current) { + return nil, fmt.Errorf("invalid previous version %q", current) + } + if cfg.BuildMod == "vendor" { + return nil, errQueryDisabled + } + if allowed == nil { + allowed = func(context.Context, module.Version) error { return nil } + } + + if path == Target.Path && (query == "upgrade" || query == "patch") { + if err := allowed(ctx, Target); err != nil { + return nil, fmt.Errorf("internal error: main module version is not allowed: %w", err) + } + return &modfetch.RevInfo{Version: Target.Version}, nil + } + + if path == "std" || path == "cmd" { + return nil, fmt.Errorf("can't query specific version (%q) of standard-library module %q", query, path) + } + + repo, err := lookupRepo(proxy, path) + if err != nil { + return nil, err + } + + // Parse query to detect parse errors (and possibly handle query) + // before any network I/O. + qm, err := newQueryMatcher(path, query, current, allowed) + if (err == nil && qm.canStat) || err == errRevQuery { + // Direct lookup of a commit identifier or complete (non-prefix) semantic + // version. + + // If the identifier is not a canonical semver tag — including if it's a + // semver tag with a +metadata suffix — then modfetch.Stat will populate + // info.Version with a suitable pseudo-version. + info, err := repo.Stat(query) + if err != nil { + queryErr := err + // The full query doesn't correspond to a tag. If it is a semantic version + // with a +metadata suffix, see if there is a tag without that suffix: + // semantic versioning defines them to be equivalent. + canonicalQuery := module.CanonicalVersion(query) + if canonicalQuery != "" && query != canonicalQuery { + info, err = repo.Stat(canonicalQuery) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return info, err + } + } + if err != nil { + return nil, queryErr + } + } + if err := allowed(ctx, module.Version{Path: path, Version: info.Version}); errors.Is(err, ErrDisallowed) { + return nil, err + } + return info, nil + } else if err != nil { + return nil, err + } + + // Load versions and execute query. + versions, err := repo.Versions(qm.prefix) + if err != nil { + return nil, err + } + releases, prereleases, err := qm.filterVersions(ctx, versions) + if err != nil { + return nil, err + } + + lookup := func(v string) (*modfetch.RevInfo, error) { + rev, err := repo.Stat(v) + if err != nil { + return nil, err + } + + if (query == "upgrade" || query == "patch") && modfetch.IsPseudoVersion(current) && !rev.Time.IsZero() { + // Don't allow "upgrade" or "patch" to move from a pseudo-version + // to a chronologically older version or pseudo-version. + // + // If the current version is a pseudo-version from an untagged branch, it + // may be semantically lower than the "latest" release or the latest + // pseudo-version on the main branch. A user on such a version is unlikely + // to intend to “upgrade” to a version that already existed at that point + // in time. + // + // We do this only if the current version is a pseudo-version: if the + // version is tagged, the author of the dependency module has given us + // explicit information about their intended precedence of this version + // relative to other versions, and we shouldn't contradict that + // information. (For example, v1.0.1 might be a backport of a fix already + // incorporated into v1.1.0, in which case v1.0.1 would be chronologically + // newer but v1.1.0 is still an “upgrade”; or v1.0.2 might be a revert of + // an unsuccessful fix in v1.0.1, in which case the v1.0.2 commit may be + // older than the v1.0.1 commit despite the tag itself being newer.) + currentTime, err := modfetch.PseudoVersionTime(current) + if err == nil && rev.Time.Before(currentTime) { + if err := allowed(ctx, module.Version{Path: path, Version: current}); errors.Is(err, ErrDisallowed) { + return nil, err + } + return repo.Stat(current) + } + } + + return rev, nil + } + + if qm.preferLower { + if len(releases) > 0 { + return lookup(releases[0]) + } + if len(prereleases) > 0 { + return lookup(prereleases[0]) + } + } else { + if len(releases) > 0 { + return lookup(releases[len(releases)-1]) + } + if len(prereleases) > 0 { + return lookup(prereleases[len(prereleases)-1]) + } + } + + if qm.mayUseLatest { + latest, err := repo.Latest() + if err == nil { + if qm.allowsVersion(ctx, latest.Version) { + return lookup(latest.Version) + } + } else if !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + } + + if (query == "upgrade" || query == "patch") && current != "" && current != "none" { + // "upgrade" and "patch" may stay on the current version if allowed. + if err := allowed(ctx, module.Version{Path: path, Version: current}); errors.Is(err, ErrDisallowed) { + return nil, err + } + return lookup(current) + } + + return nil, &NoMatchingVersionError{query: query, current: current} +} + +// IsRevisionQuery returns true if vers is a version query that may refer to +// a particular version or revision in a repository like "v1.0.0", "master", +// or "0123abcd". IsRevisionQuery returns false if vers is a query that +// chooses from among available versions like "latest" or ">v1.0.0". +func IsRevisionQuery(vers string) bool { + if vers == "latest" || + vers == "upgrade" || + vers == "patch" || + strings.HasPrefix(vers, "<") || + strings.HasPrefix(vers, ">") || + (semver.IsValid(vers) && isSemverPrefix(vers)) { + return false + } + return true +} + +// isSemverPrefix reports whether v is a semantic version prefix: v1 or v1.2 (not v1.2.3). +// The caller is assumed to have checked that semver.IsValid(v) is true. +func isSemverPrefix(v string) bool { + dots := 0 + for i := 0; i < len(v); i++ { + switch v[i] { + case '-', '+': + return false + case '.': + dots++ + if dots >= 2 { + return false + } + } + } + return true +} + +type queryMatcher struct { + path string + prefix string + filter func(version string) bool + allowed AllowedFunc + canStat bool // if true, the query can be resolved by repo.Stat + preferLower bool // if true, choose the lowest matching version + mayUseLatest bool + preferIncompatible bool +} + +var errRevQuery = errors.New("query refers to a non-semver revision") + +// newQueryMatcher returns a new queryMatcher that matches the versions +// specified by the given query on the module with the given path. +// +// If the query can only be resolved by statting a non-SemVer revision, +// newQueryMatcher returns errRevQuery. +func newQueryMatcher(path string, query, current string, allowed AllowedFunc) (*queryMatcher, error) { + badVersion := func(v string) (*queryMatcher, error) { + return nil, fmt.Errorf("invalid semantic version %q in range %q", v, query) + } + + matchesMajor := func(v string) bool { + _, pathMajor, ok := module.SplitPathVersion(path) + if !ok { + return false + } + return module.CheckPathMajor(v, pathMajor) == nil + } + + qm := &queryMatcher{ + path: path, + allowed: allowed, + preferIncompatible: strings.HasSuffix(current, "+incompatible"), + } + + switch { + case query == "latest": + qm.mayUseLatest = true + + case query == "upgrade": + if current == "" || current == "none" { + qm.mayUseLatest = true + } else { + qm.mayUseLatest = modfetch.IsPseudoVersion(current) + qm.filter = func(mv string) bool { return semver.Compare(mv, current) >= 0 } + } + + case query == "patch": + if current == "none" { + return nil, &NoPatchBaseError{path} + } + if current == "" { + qm.mayUseLatest = true + } else { + qm.mayUseLatest = modfetch.IsPseudoVersion(current) + qm.prefix = semver.MajorMinor(current) + "." + qm.filter = func(mv string) bool { return semver.Compare(mv, current) >= 0 } + } + + case strings.HasPrefix(query, "<="): + v := query[len("<="):] + if !semver.IsValid(v) { + return badVersion(v) + } + if isSemverPrefix(v) { + // Refuse to say whether <=v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). + return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) <= 0 } + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case strings.HasPrefix(query, "<"): + v := query[len("<"):] + if !semver.IsValid(v) { + return badVersion(v) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) < 0 } + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case strings.HasPrefix(query, ">="): + v := query[len(">="):] + if !semver.IsValid(v) { + return badVersion(v) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) >= 0 } + qm.preferLower = true + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case strings.HasPrefix(query, ">"): + v := query[len(">"):] + if !semver.IsValid(v) { + return badVersion(v) + } + if isSemverPrefix(v) { + // Refuse to say whether >v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). + return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) > 0 } + qm.preferLower = true + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case semver.IsValid(query): + if isSemverPrefix(query) { + qm.prefix = query + "." + // Do not allow the query "v1.2" to match versions lower than "v1.2.0", + // such as prereleases for that version. (https://golang.org/issue/31972) + qm.filter = func(mv string) bool { return semver.Compare(mv, query) >= 0 } + } else { + qm.canStat = true + qm.filter = func(mv string) bool { return semver.Compare(mv, query) == 0 } + qm.prefix = semver.Canonical(query) + } + if !matchesMajor(query) { + qm.preferIncompatible = true + } + + default: + return nil, errRevQuery + } + + return qm, nil +} + +// allowsVersion reports whether version v is allowed by the prefix, filter, and +// AllowedFunc of qm. +func (qm *queryMatcher) allowsVersion(ctx context.Context, v string) bool { + if qm.prefix != "" && !strings.HasPrefix(v, qm.prefix) { + return false + } + if qm.filter != nil && !qm.filter(v) { + return false + } + if qm.allowed != nil { + if err := qm.allowed(ctx, module.Version{Path: qm.path, Version: v}); errors.Is(err, ErrDisallowed) { + return false + } + } + return true +} + +// filterVersions classifies versions into releases and pre-releases, filtering +// out: +// 1. versions that do not satisfy the 'allowed' predicate, and +// 2. "+incompatible" versions, if a compatible one satisfies the predicate +// and the incompatible version is not preferred. +// +// If the allowed predicate returns an error not equivalent to ErrDisallowed, +// filterVersions returns that error. +func (qm *queryMatcher) filterVersions(ctx context.Context, versions []string) (releases, prereleases []string, err error) { + needIncompatible := qm.preferIncompatible + + var lastCompatible string + for _, v := range versions { + if !qm.allowsVersion(ctx, v) { + continue + } + + if !needIncompatible { + // We're not yet sure whether we need to include +incomptaible versions. + // Keep track of the last compatible version we've seen, and use the + // presence (or absence) of a go.mod file in that version to decide: a + // go.mod file implies that the module author is supporting modules at a + // compatible version (and we should ignore +incompatible versions unless + // requested explicitly), while a lack of go.mod file implies the + // potential for legacy (pre-modules) versioning without semantic import + // paths (and thus *with* +incompatible versions). + // + // This isn't strictly accurate if the latest compatible version has been + // replaced by a local file path, because we do not allow file-path + // replacements without a go.mod file: the user would have needed to add + // one. However, replacing the last compatible version while + // simultaneously expecting to upgrade implicitly to a +incompatible + // version seems like an extreme enough corner case to ignore for now. + + if !strings.HasSuffix(v, "+incompatible") { + lastCompatible = v + } else if lastCompatible != "" { + // If the latest compatible version is allowed and has a go.mod file, + // ignore any version with a higher (+incompatible) major version. (See + // https://golang.org/issue/34165.) Note that we even prefer a + // compatible pre-release over an incompatible release. + ok, err := versionHasGoMod(ctx, module.Version{Path: qm.path, Version: lastCompatible}) + if err != nil { + return nil, nil, err + } + if ok { + // The last compatible version has a go.mod file, so that's the + // highest version we're willing to consider. Don't bother even + // looking at higher versions, because they're all +incompatible from + // here onward. + break + } + + // No acceptable compatible release has a go.mod file, so the versioning + // for the module might not be module-aware, and we should respect + // legacy major-version tags. + needIncompatible = true + } + } + + if semver.Prerelease(v) != "" { + prereleases = append(prereleases, v) + } else { + releases = append(releases, v) + } + } + + return releases, prereleases, nil +} + +type QueryResult struct { + Mod module.Version + Rev *modfetch.RevInfo + Packages []string +} + +// QueryPackages is like QueryPattern, but requires that the pattern match at +// least one package and omits the non-package result (if any). +func QueryPackages(ctx context.Context, pattern, query string, current func(string) string, allowed AllowedFunc) ([]QueryResult, error) { + pkgMods, modOnly, err := QueryPattern(ctx, pattern, query, current, allowed) + + if len(pkgMods) == 0 && err == nil { + return nil, &PackageNotInModuleError{ + Mod: modOnly.Mod, + Replacement: Replacement(modOnly.Mod), + Query: query, + Pattern: pattern, + } + } + + return pkgMods, err +} + +// QueryPattern looks up the module(s) containing at least one package matching +// the given pattern at the given version. The results are sorted by module path +// length in descending order. If any proxy provides a non-empty set of candidate +// modules, no further proxies are tried. +// +// For wildcard patterns, QueryPattern looks in modules with package paths up to +// the first "..." in the pattern. For the pattern "example.com/a/b.../c", +// QueryPattern would consider prefixes of "example.com/a". +// +// If any matching package is in the main module, QueryPattern considers only +// the main module and only the version "latest", without checking for other +// possible modules. +// +// QueryPattern always returns at least one QueryResult (which may be only +// modOnly) or a non-nil error. +func QueryPattern(ctx context.Context, pattern, query string, current func(string) string, allowed AllowedFunc) (pkgMods []QueryResult, modOnly *QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.QueryPattern "+pattern+" "+query) + defer span.Done() + + base := pattern + + firstError := func(m *search.Match) error { + if len(m.Errs) == 0 { + return nil + } + return m.Errs[0] + } + + var match func(mod module.Version, root string, isLocal bool) *search.Match + matchPattern := search.MatchPattern(pattern) + + if i := strings.Index(pattern, "..."); i >= 0 { + base = pathpkg.Dir(pattern[:i+3]) + if base == "." { + return nil, nil, &WildcardInFirstElementError{Pattern: pattern, Query: query} + } + match = func(mod module.Version, root string, isLocal bool) *search.Match { + m := search.NewMatch(pattern) + matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{mod}) + return m + } + } else { + match = func(mod module.Version, root string, isLocal bool) *search.Match { + m := search.NewMatch(pattern) + prefix := mod.Path + if mod == Target { + prefix = targetPrefix + } + if _, ok, err := dirInModule(pattern, prefix, root, isLocal); err != nil { + m.AddError(err) + } else if ok { + m.Pkgs = []string{pattern} + } + return m + } + } + + var queryMatchesMainModule bool + if HasModRoot() { + m := match(Target, modRoot, true) + if len(m.Pkgs) > 0 { + if query != "upgrade" && query != "patch" { + return nil, nil, &QueryMatchesPackagesInMainModuleError{ + Pattern: pattern, + Query: query, + Packages: m.Pkgs, + } + } + if err := allowed(ctx, Target); err != nil { + return nil, nil, fmt.Errorf("internal error: package %s is in the main module (%s), but version is not allowed: %w", pattern, Target.Path, err) + } + return []QueryResult{{ + Mod: Target, + Rev: &modfetch.RevInfo{Version: Target.Version}, + Packages: m.Pkgs, + }}, nil, nil + } + if err := firstError(m); err != nil { + return nil, nil, err + } + + if matchPattern(Target.Path) { + queryMatchesMainModule = true + } + + if (query == "upgrade" || query == "patch") && queryMatchesMainModule { + if err := allowed(ctx, Target); err == nil { + modOnly = &QueryResult{ + Mod: Target, + Rev: &modfetch.RevInfo{Version: Target.Version}, + } + } + } + } + + var ( + results []QueryResult + candidateModules = modulePrefixesExcludingTarget(base) + ) + if len(candidateModules) == 0 { + if modOnly != nil { + return nil, modOnly, nil + } else if queryMatchesMainModule { + return nil, nil, &QueryMatchesMainModuleError{ + Pattern: pattern, + Query: query, + } + } else { + return nil, nil, &PackageNotInModuleError{ + Mod: Target, + Query: query, + Pattern: pattern, + } + } + } + + err = modfetch.TryProxies(func(proxy string) error { + queryModule := func(ctx context.Context, path string) (r QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.QueryPattern.queryModule ["+proxy+"] "+path) + defer span.Done() + + pathCurrent := current(path) + r.Mod.Path = path + r.Rev, err = queryProxy(ctx, proxy, path, query, pathCurrent, allowed) + if err != nil { + return r, err + } + r.Mod.Version = r.Rev.Version + needSum := true + root, isLocal, err := fetch(ctx, r.Mod, needSum) + if err != nil { + return r, err + } + m := match(r.Mod, root, isLocal) + r.Packages = m.Pkgs + if len(r.Packages) == 0 && !matchPattern(path) { + if err := firstError(m); err != nil { + return r, err + } + return r, &PackageNotInModuleError{ + Mod: r.Mod, + Replacement: Replacement(r.Mod), + Query: query, + Pattern: pattern, + } + } + return r, nil + } + + allResults, err := queryPrefixModules(ctx, candidateModules, queryModule) + results = allResults[:0] + for _, r := range allResults { + if len(r.Packages) == 0 { + modOnly = &r + } else { + results = append(results, r) + } + } + return err + }) + + if queryMatchesMainModule && len(results) == 0 && modOnly == nil && errors.Is(err, fs.ErrNotExist) { + return nil, nil, &QueryMatchesMainModuleError{ + Pattern: pattern, + Query: query, + } + } + return results[:len(results):len(results)], modOnly, err +} + +// modulePrefixesExcludingTarget returns all prefixes of path that may plausibly +// exist as a module, excluding targetPrefix but otherwise including path +// itself, sorted by descending length. +func modulePrefixesExcludingTarget(path string) []string { + prefixes := make([]string, 0, strings.Count(path, "/")+1) + + for { + if path != targetPrefix { + if _, _, ok := module.SplitPathVersion(path); ok { + prefixes = append(prefixes, path) + } + } + + j := strings.LastIndexByte(path, '/') + if j < 0 { + break + } + path = path[:j] + } + + return prefixes +} + +func queryPrefixModules(ctx context.Context, candidateModules []string, queryModule func(ctx context.Context, path string) (QueryResult, error)) (found []QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.queryPrefixModules") + defer span.Done() + + // If the path we're attempting is not in the module cache and we don't have a + // fetch result cached either, we'll end up making a (potentially slow) + // request to the proxy or (often even slower) the origin server. + // To minimize latency, execute all of those requests in parallel. + type result struct { + QueryResult + err error + } + results := make([]result, len(candidateModules)) + var wg sync.WaitGroup + wg.Add(len(candidateModules)) + for i, p := range candidateModules { + ctx := trace.StartGoroutine(ctx) + go func(p string, r *result) { + r.QueryResult, r.err = queryModule(ctx, p) + wg.Done() + }(p, &results[i]) + } + wg.Wait() + + // Classify the results. In case of failure, identify the error that the user + // is most likely to find helpful: the most useful class of error at the + // longest matching path. + var ( + noPackage *PackageNotInModuleError + noVersion *NoMatchingVersionError + noPatchBase *NoPatchBaseError + notExistErr error + ) + for _, r := range results { + switch rErr := r.err.(type) { + case nil: + found = append(found, r.QueryResult) + case *PackageNotInModuleError: + // Given the option, prefer to attribute “package not in module” + // to modules other than the main one. + if noPackage == nil || noPackage.Mod == Target { + noPackage = rErr + } + case *NoMatchingVersionError: + if noVersion == nil { + noVersion = rErr + } + case *NoPatchBaseError: + if noPatchBase == nil { + noPatchBase = rErr + } + default: + if errors.Is(rErr, fs.ErrNotExist) { + if notExistErr == nil { + notExistErr = rErr + } + } else if err == nil { + if len(found) > 0 || noPackage != nil { + // golang.org/issue/34094: If we have already found a module that + // could potentially contain the target package, ignore unclassified + // errors for modules with shorter paths. + + // golang.org/issue/34383 is a special case of this: if we have + // already found example.com/foo/v2@v2.0.0 with a matching go.mod + // file, ignore the error from example.com/foo@v2.0.0. + } else { + err = r.err + } + } + } + } + + // TODO(#26232): If len(found) == 0 and some of the errors are 4xx HTTP + // codes, have the auth package recheck the failed paths. + // If we obtain new credentials for any of them, re-run the above loop. + + if len(found) == 0 && err == nil { + switch { + case noPackage != nil: + err = noPackage + case noVersion != nil: + err = noVersion + case noPatchBase != nil: + err = noPatchBase + case notExistErr != nil: + err = notExistErr + default: + panic("queryPrefixModules: no modules found, but no error detected") + } + } + + return found, err +} + +// A NoMatchingVersionError indicates that Query found a module at the requested +// path, but not at any versions satisfying the query string and allow-function. +// +// NOTE: NoMatchingVersionError MUST NOT implement Is(fs.ErrNotExist). +// +// If the module came from a proxy, that proxy had to return a successful status +// code for the versions it knows about, and thus did not have the opportunity +// to return a non-400 status code to suppress fallback. +type NoMatchingVersionError struct { + query, current string +} + +func (e *NoMatchingVersionError) Error() string { + currentSuffix := "" + if (e.query == "upgrade" || e.query == "patch") && e.current != "" && e.current != "none" { + currentSuffix = fmt.Sprintf(" (current version is %s)", e.current) + } + return fmt.Sprintf("no matching versions for query %q", e.query) + currentSuffix +} + +// A NoPatchBaseError indicates that Query was called with the query "patch" +// but with a current version of "" or "none". +type NoPatchBaseError struct { + path string +} + +func (e *NoPatchBaseError) Error() string { + return fmt.Sprintf(`can't query version "patch" of module %s: no existing version is required`, e.path) +} + +// A WildcardInFirstElementError indicates that a pattern passed to QueryPattern +// had a wildcard in its first path element, and therefore had no pattern-prefix +// modules to search in. +type WildcardInFirstElementError struct { + Pattern string + Query string +} + +func (e *WildcardInFirstElementError) Error() string { + return fmt.Sprintf("no modules to query for %s@%s because first path element contains a wildcard", e.Pattern, e.Query) +} + +// A PackageNotInModuleError indicates that QueryPattern found a candidate +// module at the requested version, but that module did not contain any packages +// matching the requested pattern. +// +// NOTE: PackageNotInModuleError MUST NOT implement Is(fs.ErrNotExist). +// +// If the module came from a proxy, that proxy had to return a successful status +// code for the versions it knows about, and thus did not have the opportunity +// to return a non-400 status code to suppress fallback. +type PackageNotInModuleError struct { + Mod module.Version + Replacement module.Version + Query string + Pattern string +} + +func (e *PackageNotInModuleError) Error() string { + if e.Mod == Target { + if strings.Contains(e.Pattern, "...") { + return fmt.Sprintf("main module (%s) does not contain packages matching %s", Target.Path, e.Pattern) + } + return fmt.Sprintf("main module (%s) does not contain package %s", Target.Path, e.Pattern) + } + + found := "" + if r := e.Replacement; r.Path != "" { + replacement := r.Path + if r.Version != "" { + replacement = fmt.Sprintf("%s@%s", r.Path, r.Version) + } + if e.Query == e.Mod.Version { + found = fmt.Sprintf(" (replaced by %s)", replacement) + } else { + found = fmt.Sprintf(" (%s, replaced by %s)", e.Mod.Version, replacement) + } + } else if e.Query != e.Mod.Version { + found = fmt.Sprintf(" (%s)", e.Mod.Version) + } + + if strings.Contains(e.Pattern, "...") { + return fmt.Sprintf("module %s@%s found%s, but does not contain packages matching %s", e.Mod.Path, e.Query, found, e.Pattern) + } + return fmt.Sprintf("module %s@%s found%s, but does not contain package %s", e.Mod.Path, e.Query, found, e.Pattern) +} + +func (e *PackageNotInModuleError) ImportPath() string { + if !strings.Contains(e.Pattern, "...") { + return e.Pattern + } + return "" +} + +// ModuleHasRootPackage returns whether module m contains a package m.Path. +func ModuleHasRootPackage(ctx context.Context, m module.Version) (bool, error) { + needSum := false + root, isLocal, err := fetch(ctx, m, needSum) + if err != nil { + return false, err + } + _, ok, err := dirInModule(m.Path, m.Path, root, isLocal) + return ok, err +} + +func versionHasGoMod(ctx context.Context, m module.Version) (bool, error) { + needSum := false + root, _, err := fetch(ctx, m, needSum) + if err != nil { + return false, err + } + fi, err := os.Stat(filepath.Join(root, "go.mod")) + return err == nil && !fi.IsDir(), nil +} + +// A versionRepo is a subset of modfetch.Repo that can report information about +// available versions, but cannot fetch specific source files. +type versionRepo interface { + ModulePath() string + Versions(prefix string) ([]string, error) + Stat(rev string) (*modfetch.RevInfo, error) + Latest() (*modfetch.RevInfo, error) +} + +var _ versionRepo = modfetch.Repo(nil) + +func lookupRepo(proxy, path string) (repo versionRepo, err error) { + err = module.CheckPath(path) + if err == nil { + repo = modfetch.Lookup(proxy, path) + } else { + repo = emptyRepo{path: path, err: err} + } + + if index == nil { + return repo, err + } + if _, ok := index.highestReplaced[path]; !ok { + return repo, err + } + + return &replacementRepo{repo: repo}, nil +} + +// An emptyRepo is a versionRepo that contains no versions. +type emptyRepo struct { + path string + err error +} + +var _ versionRepo = emptyRepo{} + +func (er emptyRepo) ModulePath() string { return er.path } +func (er emptyRepo) Versions(prefix string) ([]string, error) { return nil, nil } +func (er emptyRepo) Stat(rev string) (*modfetch.RevInfo, error) { return nil, er.err } +func (er emptyRepo) Latest() (*modfetch.RevInfo, error) { return nil, er.err } + +// A replacementRepo augments a versionRepo to include the replacement versions +// (if any) found in the main module's go.mod file. +// +// A replacementRepo suppresses "not found" errors for otherwise-nonexistent +// modules, so a replacementRepo should only be constructed for a module that +// actually has one or more valid replacements. +type replacementRepo struct { + repo versionRepo +} + +var _ versionRepo = (*replacementRepo)(nil) + +func (rr *replacementRepo) ModulePath() string { return rr.repo.ModulePath() } + +// Versions returns the versions from rr.repo augmented with any matching +// replacement versions. +func (rr *replacementRepo) Versions(prefix string) ([]string, error) { + repoVersions, err := rr.repo.Versions(prefix) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + + versions := repoVersions + if index != nil && len(index.replace) > 0 { + path := rr.ModulePath() + for m, _ := range index.replace { + if m.Path == path && strings.HasPrefix(m.Version, prefix) && m.Version != "" && !modfetch.IsPseudoVersion(m.Version) { + versions = append(versions, m.Version) + } + } + } + + if len(versions) == len(repoVersions) { // No replacement versions added. + return versions, nil + } + + sort.Slice(versions, func(i, j int) bool { + return semver.Compare(versions[i], versions[j]) < 0 + }) + str.Uniq(&versions) + return versions, nil +} + +func (rr *replacementRepo) Stat(rev string) (*modfetch.RevInfo, error) { + info, err := rr.repo.Stat(rev) + if err == nil || index == nil || len(index.replace) == 0 { + return info, err + } + + v := module.CanonicalVersion(rev) + if v != rev { + // The replacements in the go.mod file list only canonical semantic versions, + // so a non-canonical version can't possibly have a replacement. + return info, err + } + + path := rr.ModulePath() + _, pathMajor, ok := module.SplitPathVersion(path) + if ok && pathMajor == "" { + if err := module.CheckPathMajor(v, pathMajor); err != nil && semver.Build(v) == "" { + v += "+incompatible" + } + } + + if r := Replacement(module.Version{Path: path, Version: v}); r.Path == "" { + return info, err + } + return rr.replacementStat(v) +} + +func (rr *replacementRepo) Latest() (*modfetch.RevInfo, error) { + info, err := rr.repo.Latest() + + if index != nil { + path := rr.ModulePath() + if v, ok := index.highestReplaced[path]; ok { + if v == "" { + // The only replacement is a wildcard that doesn't specify a version, so + // synthesize a pseudo-version with an appropriate major version and a + // timestamp below any real timestamp. That way, if the main module is + // used from within some other module, the user will be able to upgrade + // the requirement to any real version they choose. + if _, pathMajor, ok := module.SplitPathVersion(path); ok && len(pathMajor) > 0 { + v = modfetch.PseudoVersion(pathMajor[1:], "", time.Time{}, "000000000000") + } else { + v = modfetch.PseudoVersion("v0", "", time.Time{}, "000000000000") + } + } + + if err != nil || semver.Compare(v, info.Version) > 0 { + return rr.replacementStat(v) + } + } + } + + return info, err +} + +func (rr *replacementRepo) replacementStat(v string) (*modfetch.RevInfo, error) { + rev := &modfetch.RevInfo{Version: v} + if modfetch.IsPseudoVersion(v) { + rev.Time, _ = modfetch.PseudoVersionTime(v) + rev.Short, _ = modfetch.PseudoVersionRev(v) + } + return rev, nil +} + +// A QueryMatchesMainModuleError indicates that a query requests +// a version of the main module that cannot be satisfied. +// (The main module's version cannot be changed.) +type QueryMatchesMainModuleError struct { + Pattern string + Query string +} + +func (e *QueryMatchesMainModuleError) Error() string { + if e.Pattern == Target.Path { + return fmt.Sprintf("can't request version %q of the main module (%s)", e.Query, e.Pattern) + } + + return fmt.Sprintf("can't request version %q of pattern %q that includes the main module (%s)", e.Query, e.Pattern, Target.Path) +} + +// A QueryMatchesPackagesInMainModuleError indicates that a query cannot be +// satisfied because it matches one or more packages found in the main module. +type QueryMatchesPackagesInMainModuleError struct { + Pattern string + Query string + Packages []string +} + +func (e *QueryMatchesPackagesInMainModuleError) Error() string { + if len(e.Packages) > 1 { + return fmt.Sprintf("pattern %s matches %d packages in the main module, so can't request version %s", e.Pattern, len(e.Packages), e.Query) + } + + if search.IsMetaPackage(e.Pattern) || strings.Contains(e.Pattern, "...") { + return fmt.Sprintf("pattern %s matches package %s in the main module, so can't request version %s", e.Pattern, e.Packages[0], e.Query) + } + + return fmt.Sprintf("package %s is in the main module, so can't request version %s", e.Packages[0], e.Query) +} diff --git a/src/cmd/go/internal/modload/query_test.go b/src/cmd/go/internal/modload/query_test.go new file mode 100644 index 0000000..e225a0e --- /dev/null +++ b/src/cmd/go/internal/modload/query_test.go @@ -0,0 +1,216 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "context" + "internal/testenv" + "log" + "os" + "path" + "path/filepath" + "strings" + "testing" + + "cmd/go/internal/cfg" + + "golang.org/x/mod/module" +) + +func TestMain(m *testing.M) { + os.Exit(testMain(m)) +} + +func testMain(m *testing.M) int { + cfg.GOPROXY = "direct" + + dir, err := os.MkdirTemp("", "modload-test-") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(dir) + + os.Setenv("GOPATH", dir) + cfg.BuildContext.GOPATH = dir + cfg.GOMODCACHE = filepath.Join(dir, "pkg/mod") + return m.Run() +} + +var ( + queryRepo = "vcs-test.golang.org/git/querytest.git" + queryRepoV2 = queryRepo + "/v2" + queryRepoV3 = queryRepo + "/v3" + + // Empty version list (no semver tags), not actually empty. + emptyRepoPath = "vcs-test.golang.org/git/emptytest.git" +) + +var queryTests = []struct { + path string + query string + current string + allow string + vers string + err string +}{ + /* + git init + echo module vcs-test.golang.org/git/querytest.git >go.mod + git add go.mod + git commit -m v1 go.mod + git tag start + for i in v0.0.0-pre1 v0.0.0 v0.0.1 v0.0.2 v0.0.3 v0.1.0 v0.1.1 v0.1.2 v0.3.0 v1.0.0 v1.1.0 v1.9.0 v1.9.9 v1.9.10-pre1 v1.9.10-pre2+metadata unversioned; do + echo before $i >status + git add status + git commit -m "before $i" status + echo at $i >status + git commit -m "at $i" status + git tag $i + done + git tag favorite v0.0.3 + + git branch v2 start + git checkout v2 + echo module vcs-test.golang.org/git/querytest.git/v2 >go.mod + git commit -m v2 go.mod + for i in v2.0.0 v2.1.0 v2.2.0 v2.5.5 v2.6.0-pre1; do + echo before $i >status + git add status + git commit -m "before $i" status + echo at $i >status + git commit -m "at $i" status + git tag $i + done + git checkout v2.5.5 + echo after v2.5.5 >status + git commit -m 'after v2.5.5' status + git checkout master + zip -r ../querytest.zip + gsutil cp ../querytest.zip gs://vcs-test/git/querytest.zip + curl 'https://vcs-test.golang.org/git/querytest?go-get=1' + */ + {path: queryRepo, query: "<v0.0.0", vers: "v0.0.0-pre1"}, + {path: queryRepo, query: "<v0.0.0-pre1", err: `no matching versions for query "<v0.0.0-pre1"`}, + {path: queryRepo, query: "<=v0.0.0", vers: "v0.0.0"}, + {path: queryRepo, query: ">v0.0.0", vers: "v0.0.1"}, + {path: queryRepo, query: ">=v0.0.0", vers: "v0.0.0"}, + {path: queryRepo, query: "v0.0.1", vers: "v0.0.1"}, + {path: queryRepo, query: "v0.0.1+foo", vers: "v0.0.1"}, + {path: queryRepo, query: "v0.0.99", err: `vcs-test.golang.org/git/querytest.git@v0.0.99: invalid version: unknown revision v0.0.99`}, + {path: queryRepo, query: "v0", vers: "v0.3.0"}, + {path: queryRepo, query: "v0.1", vers: "v0.1.2"}, + {path: queryRepo, query: "v0.2", err: `no matching versions for query "v0.2"`}, + {path: queryRepo, query: "v0.0", vers: "v0.0.3"}, + {path: queryRepo, query: "v1.9.10-pre2+metadata", vers: "v1.9.10-pre2.0.20190513201126-42abcb6df8ee"}, + {path: queryRepo, query: "ed5ffdaa", vers: "v1.9.10-pre2.0.20191220134614-ed5ffdaa1f5e"}, + + // golang.org/issue/29262: The major version for for a module without a suffix + // should be based on the most recent tag (v1 as appropriate, not v0 + // unconditionally). + {path: queryRepo, query: "42abcb6df8ee", vers: "v1.9.10-pre2.0.20190513201126-42abcb6df8ee"}, + + {path: queryRepo, query: "v1.9.10-pre2+wrongmetadata", err: `vcs-test.golang.org/git/querytest.git@v1.9.10-pre2+wrongmetadata: invalid version: unknown revision v1.9.10-pre2+wrongmetadata`}, + {path: queryRepo, query: "v1.9.10-pre2", err: `vcs-test.golang.org/git/querytest.git@v1.9.10-pre2: invalid version: unknown revision v1.9.10-pre2`}, + {path: queryRepo, query: "latest", vers: "v1.9.9"}, + {path: queryRepo, query: "latest", current: "v1.9.10-pre1", vers: "v1.9.9"}, + {path: queryRepo, query: "upgrade", vers: "v1.9.9"}, + {path: queryRepo, query: "upgrade", current: "v1.9.10-pre1", vers: "v1.9.10-pre1"}, + {path: queryRepo, query: "upgrade", current: "v1.9.10-pre2+metadata", vers: "v1.9.10-pre2.0.20190513201126-42abcb6df8ee"}, + {path: queryRepo, query: "upgrade", current: "v0.0.0-20190513201126-42abcb6df8ee", vers: "v0.0.0-20190513201126-42abcb6df8ee"}, + {path: queryRepo, query: "upgrade", allow: "NOMATCH", err: `no matching versions for query "upgrade"`}, + {path: queryRepo, query: "upgrade", current: "v1.9.9", allow: "NOMATCH", err: `vcs-test.golang.org/git/querytest.git@v1.9.9: disallowed module version`}, + {path: queryRepo, query: "upgrade", current: "v1.99.99", err: `vcs-test.golang.org/git/querytest.git@v1.99.99: invalid version: unknown revision v1.99.99`}, + {path: queryRepo, query: "patch", current: "", vers: "v1.9.9"}, + {path: queryRepo, query: "patch", current: "v0.1.0", vers: "v0.1.2"}, + {path: queryRepo, query: "patch", current: "v1.9.0", vers: "v1.9.9"}, + {path: queryRepo, query: "patch", current: "v1.9.10-pre1", vers: "v1.9.10-pre1"}, + {path: queryRepo, query: "patch", current: "v1.9.10-pre2+metadata", vers: "v1.9.10-pre2.0.20190513201126-42abcb6df8ee"}, + {path: queryRepo, query: "patch", current: "v1.99.99", err: `vcs-test.golang.org/git/querytest.git@v1.99.99: invalid version: unknown revision v1.99.99`}, + {path: queryRepo, query: ">v1.9.9", vers: "v1.9.10-pre1"}, + {path: queryRepo, query: ">v1.10.0", err: `no matching versions for query ">v1.10.0"`}, + {path: queryRepo, query: ">=v1.10.0", err: `no matching versions for query ">=v1.10.0"`}, + {path: queryRepo, query: "6cf84eb", vers: "v0.0.2-0.20180704023347-6cf84ebaea54"}, + + // golang.org/issue/27173: A pseudo-version may be based on the highest tag on + // any parent commit, or any existing semantically-lower tag: a given commit + // could have been a pre-release for a backport tag at any point. + {path: queryRepo, query: "3ef0cec634e0", vers: "v0.1.2-0.20180704023347-3ef0cec634e0"}, + {path: queryRepo, query: "v0.1.2-0.20180704023347-3ef0cec634e0", vers: "v0.1.2-0.20180704023347-3ef0cec634e0"}, + {path: queryRepo, query: "v0.1.1-0.20180704023347-3ef0cec634e0", vers: "v0.1.1-0.20180704023347-3ef0cec634e0"}, + {path: queryRepo, query: "v0.0.4-0.20180704023347-3ef0cec634e0", vers: "v0.0.4-0.20180704023347-3ef0cec634e0"}, + + // Invalid tags are tested in cmd/go/testdata/script/mod_pseudo_invalid.txt. + + {path: queryRepo, query: "start", vers: "v0.0.0-20180704023101-5e9e31667ddf"}, + {path: queryRepo, query: "5e9e31667ddf", vers: "v0.0.0-20180704023101-5e9e31667ddf"}, + {path: queryRepo, query: "v0.0.0-20180704023101-5e9e31667ddf", vers: "v0.0.0-20180704023101-5e9e31667ddf"}, + + {path: queryRepo, query: "7a1b6bf", vers: "v0.1.0"}, + + {path: queryRepoV2, query: "<v0.0.0", err: `no matching versions for query "<v0.0.0"`}, + {path: queryRepoV2, query: "<=v0.0.0", err: `no matching versions for query "<=v0.0.0"`}, + {path: queryRepoV2, query: ">v0.0.0", vers: "v2.0.0"}, + {path: queryRepoV2, query: ">=v0.0.0", vers: "v2.0.0"}, + + {path: queryRepoV2, query: "v2", vers: "v2.5.5"}, + {path: queryRepoV2, query: "v2.5", vers: "v2.5.5"}, + {path: queryRepoV2, query: "v2.6", err: `no matching versions for query "v2.6"`}, + {path: queryRepoV2, query: "v2.6.0-pre1", vers: "v2.6.0-pre1"}, + {path: queryRepoV2, query: "latest", vers: "v2.5.5"}, + + // Commit e0cf3de987e6 is actually v1.19.10-pre1, not anything resembling v3, + // and it has a go.mod file with a non-v3 module path. Attempting to query it + // as the v3 module should fail. + {path: queryRepoV3, query: "e0cf3de987e6", err: `vcs-test.golang.org/git/querytest.git/v3@v3.0.0-20180704024501-e0cf3de987e6: invalid version: go.mod has non-.../v3 module path "vcs-test.golang.org/git/querytest.git" (and .../v3/go.mod does not exist) at revision e0cf3de987e6`}, + + // The querytest repo does not have any commits tagged with major version 3, + // and the latest commit in the repo has a go.mod file specifying a non-v3 path. + // That should prevent us from resolving any version for the /v3 path. + {path: queryRepoV3, query: "latest", err: `no matching versions for query "latest"`}, + + {path: emptyRepoPath, query: "latest", vers: "v0.0.0-20180704023549-7bb914627242"}, + {path: emptyRepoPath, query: ">v0.0.0", err: `no matching versions for query ">v0.0.0"`}, + {path: emptyRepoPath, query: "<v10.0.0", err: `no matching versions for query "<v10.0.0"`}, +} + +func TestQuery(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + testenv.MustHaveExecPath(t, "git") + + ctx := context.Background() + + for _, tt := range queryTests { + allow := tt.allow + if allow == "" { + allow = "*" + } + allowed := func(ctx context.Context, m module.Version) error { + if ok, _ := path.Match(allow, m.Version); !ok { + return module.VersionError(m, ErrDisallowed) + } + return nil + } + tt := tt + t.Run(strings.ReplaceAll(tt.path, "/", "_")+"/"+tt.query+"/"+tt.current+"/"+allow, func(t *testing.T) { + t.Parallel() + + info, err := Query(ctx, tt.path, tt.query, tt.current, allowed) + if tt.err != "" { + if err == nil { + t.Errorf("Query(_, %q, %q, %q, %v) = %v, want error %q", tt.path, tt.query, tt.current, allow, info.Version, tt.err) + } else if err.Error() != tt.err { + t.Errorf("Query(_, %q, %q, %q, %v): %v\nwant error %q", tt.path, tt.query, tt.current, allow, err, tt.err) + } + return + } + if err != nil { + t.Fatalf("Query(_, %q, %q, %q, %v): %v\nwant %v", tt.path, tt.query, tt.current, allow, err, tt.vers) + } + if info.Version != tt.vers { + t.Errorf("Query(_, %q, %q, %q, %v) = %v, want %v", tt.path, tt.query, tt.current, allow, info.Version, tt.vers) + } + }) + } +} diff --git a/src/cmd/go/internal/modload/search.go b/src/cmd/go/internal/modload/search.go new file mode 100644 index 0000000..1fe742d --- /dev/null +++ b/src/cmd/go/internal/modload/search.go @@ -0,0 +1,219 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/imports" + "cmd/go/internal/search" + + "golang.org/x/mod/module" +) + +type stdFilter int8 + +const ( + omitStd = stdFilter(iota) + includeStd +) + +// matchPackages is like m.MatchPackages, but uses a local variable (rather than +// a global) for tags, can include or exclude packages in the standard library, +// and is restricted to the given list of modules. +func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) { + m.Pkgs = []string{} + + isMatch := func(string) bool { return true } + treeCanMatch := func(string) bool { return true } + if !m.IsMeta() { + isMatch = search.MatchPattern(m.Pattern()) + treeCanMatch = search.TreeCanMatchPattern(m.Pattern()) + } + + have := map[string]bool{ + "builtin": true, // ignore pseudo-package that exists only for documentation + } + if !cfg.BuildContext.CgoEnabled { + have["runtime/cgo"] = true // ignore during walk + } + + type pruning int8 + const ( + pruneVendor = pruning(1 << iota) + pruneGoMod + ) + + walkPkgs := func(root, importPathRoot string, prune pruning) { + root = filepath.Clean(root) + err := fsys.Walk(root, func(path string, fi fs.FileInfo, err error) error { + if err != nil { + m.AddError(err) + return nil + } + + want := true + elem := "" + + // Don't use GOROOT/src but do walk down into it. + if path == root { + if importPathRoot == "" { + return nil + } + } else { + // Avoid .foo, _foo, and testdata subdirectory trees. + _, elem = filepath.Split(path) + if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" { + want = false + } + } + + name := importPathRoot + filepath.ToSlash(path[len(root):]) + if importPathRoot == "" { + name = name[1:] // cut leading slash + } + if !treeCanMatch(name) { + want = false + } + + if !fi.IsDir() { + if fi.Mode()&fs.ModeSymlink != 0 && want { + if target, err := fsys.Stat(path); err == nil && target.IsDir() { + fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path) + } + } + return nil + } + + if !want { + return filepath.SkipDir + } + // Stop at module boundaries. + if (prune&pruneGoMod != 0) && path != root { + if fi, err := os.Stat(filepath.Join(path, "go.mod")); err == nil && !fi.IsDir() { + return filepath.SkipDir + } + } + + if !have[name] { + have[name] = true + if isMatch(name) { + if _, _, err := scanDir(path, tags); err != imports.ErrNoGo { + m.Pkgs = append(m.Pkgs, name) + } + } + } + + if elem == "vendor" && (prune&pruneVendor != 0) { + return filepath.SkipDir + } + return nil + }) + if err != nil { + m.AddError(err) + } + } + + if filter == includeStd { + walkPkgs(cfg.GOROOTsrc, "", pruneGoMod) + if treeCanMatch("cmd") { + walkPkgs(filepath.Join(cfg.GOROOTsrc, "cmd"), "cmd", pruneGoMod) + } + } + + if cfg.BuildMod == "vendor" { + if HasModRoot() { + walkPkgs(ModRoot(), targetPrefix, pruneGoMod|pruneVendor) + walkPkgs(filepath.Join(ModRoot(), "vendor"), "", pruneVendor) + } + return + } + + for _, mod := range modules { + if !treeCanMatch(mod.Path) { + continue + } + + var ( + root, modPrefix string + isLocal bool + ) + if mod == Target { + if !HasModRoot() { + continue // If there is no main module, we can't search in it. + } + root = ModRoot() + modPrefix = targetPrefix + isLocal = true + } else { + var err error + const needSum = true + root, isLocal, err = fetch(ctx, mod, needSum) + if err != nil { + m.AddError(err) + continue + } + modPrefix = mod.Path + } + + prune := pruneVendor + if isLocal { + prune |= pruneGoMod + } + walkPkgs(root, modPrefix, prune) + } + + return +} + +// MatchInModule identifies the packages matching the given pattern within the +// given module version, which does not need to be in the build list or module +// requirement graph. +// +// If m is the zero module.Version, MatchInModule matches the pattern +// against the standard library (std and cmd) in GOROOT/src. +func MatchInModule(ctx context.Context, pattern string, m module.Version, tags map[string]bool) *search.Match { + match := search.NewMatch(pattern) + if m == (module.Version{}) { + matchPackages(ctx, match, tags, includeStd, nil) + } + + LoadModFile(ctx) + + if !match.IsLiteral() { + matchPackages(ctx, match, tags, omitStd, []module.Version{m}) + return match + } + + const needSum = true + root, isLocal, err := fetch(ctx, m, needSum) + if err != nil { + match.Errs = []error{err} + return match + } + + dir, haveGoFiles, err := dirInModule(pattern, m.Path, root, isLocal) + if err != nil { + match.Errs = []error{err} + return match + } + if haveGoFiles { + if _, _, err := scanDir(dir, tags); err != imports.ErrNoGo { + // ErrNoGo indicates that the directory is not actually a Go package, + // perhaps due to the tags in use. Any other non-nil error indicates a + // problem with one or more of the Go source files, but such an error does + // not stop the package from existing, so it has no impact on matching. + match.Pkgs = []string{pattern} + } + } + return match +} diff --git a/src/cmd/go/internal/modload/stat_openfile.go b/src/cmd/go/internal/modload/stat_openfile.go new file mode 100644 index 0000000..5842b85 --- /dev/null +++ b/src/cmd/go/internal/modload/stat_openfile.go @@ -0,0 +1,28 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build js,wasm plan9 + +// On plan9, per http://9p.io/magic/man2html/2/access: “Since file permissions +// are checked by the server and group information is not known to the client, +// access must open the file to check permissions.” +// +// aix and js,wasm are similar, in that they do not define syscall.Access. + +package modload + +import ( + "io/fs" + "os" +) + +// hasWritePerm reports whether the current user has permission to write to the +// file with the given info. +func hasWritePerm(path string, _ fs.FileInfo) bool { + if f, err := os.OpenFile(path, os.O_WRONLY, 0); err == nil { + f.Close() + return true + } + return false +} diff --git a/src/cmd/go/internal/modload/stat_unix.go b/src/cmd/go/internal/modload/stat_unix.go new file mode 100644 index 0000000..f49278e --- /dev/null +++ b/src/cmd/go/internal/modload/stat_unix.go @@ -0,0 +1,32 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris + +package modload + +import ( + "io/fs" + "os" + "syscall" +) + +// hasWritePerm reports whether the current user has permission to write to the +// file with the given info. +// +// Although the root user on most Unix systems can write to files even without +// permission, hasWritePerm reports false if no appropriate permission bit is +// set even if the current user is root. +func hasWritePerm(path string, fi fs.FileInfo) bool { + if os.Getuid() == 0 { + // The root user can access any file, but we still want to default to + // read-only mode if the go.mod file is marked as globally non-writable. + // (If the user really intends not to be in readonly mode, they can + // pass -mod=mod explicitly.) + return fi.Mode()&0222 != 0 + } + + const W_OK = 0x2 + return syscall.Access(path, W_OK) == nil +} diff --git a/src/cmd/go/internal/modload/stat_windows.go b/src/cmd/go/internal/modload/stat_windows.go new file mode 100644 index 0000000..0ac2391 --- /dev/null +++ b/src/cmd/go/internal/modload/stat_windows.go @@ -0,0 +1,21 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build windows + +package modload + +import "io/fs" + +// hasWritePerm reports whether the current user has permission to write to the +// file with the given info. +func hasWritePerm(_ string, fi fs.FileInfo) bool { + // Windows has a read-only attribute independent of ACLs, so use that to + // determine whether the file is intended to be overwritten. + // + // Per https://golang.org/pkg/os/#Chmod: + // “On Windows, only the 0200 bit (owner writable) of mode is used; it + // controls whether the file's read-only attribute is set or cleared.” + return fi.Mode()&0200 != 0 +} diff --git a/src/cmd/go/internal/modload/vendor.go b/src/cmd/go/internal/modload/vendor.go new file mode 100644 index 0000000..d8fd91f --- /dev/null +++ b/src/cmd/go/internal/modload/vendor.go @@ -0,0 +1,219 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "sync" + + "cmd/go/internal/base" + + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +var ( + vendorOnce sync.Once + vendorList []module.Version // modules that contribute packages to the build, in order of appearance + vendorReplaced []module.Version // all replaced modules; may or may not also contribute packages + vendorVersion map[string]string // module path → selected version (if known) + vendorPkgModule map[string]module.Version // package → containing module + vendorMeta map[module.Version]vendorMetadata +) + +type vendorMetadata struct { + Explicit bool + Replacement module.Version +} + +// readVendorList reads the list of vendored modules from vendor/modules.txt. +func readVendorList() { + vendorOnce.Do(func() { + vendorList = nil + vendorPkgModule = make(map[string]module.Version) + vendorVersion = make(map[string]string) + vendorMeta = make(map[module.Version]vendorMetadata) + data, err := os.ReadFile(filepath.Join(ModRoot(), "vendor/modules.txt")) + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + base.Fatalf("go: %s", err) + } + return + } + + var mod module.Version + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, "# ") { + f := strings.Fields(line) + + if len(f) < 3 { + continue + } + if semver.IsValid(f[2]) { + // A module, but we don't yet know whether it is in the build list or + // only included to indicate a replacement. + mod = module.Version{Path: f[1], Version: f[2]} + f = f[3:] + } else if f[2] == "=>" { + // A wildcard replacement found in the main module's go.mod file. + mod = module.Version{Path: f[1]} + f = f[2:] + } else { + // Not a version or a wildcard replacement. + // We don't know how to interpret this module line, so ignore it. + mod = module.Version{} + continue + } + + if len(f) >= 2 && f[0] == "=>" { + meta := vendorMeta[mod] + if len(f) == 2 { + // File replacement. + meta.Replacement = module.Version{Path: f[1]} + vendorReplaced = append(vendorReplaced, mod) + } else if len(f) == 3 && semver.IsValid(f[2]) { + // Path and version replacement. + meta.Replacement = module.Version{Path: f[1], Version: f[2]} + vendorReplaced = append(vendorReplaced, mod) + } else { + // We don't understand this replacement. Ignore it. + } + vendorMeta[mod] = meta + } + continue + } + + // Not a module line. Must be a package within a module or a metadata + // directive, either of which requires a preceding module line. + if mod.Path == "" { + continue + } + + if strings.HasPrefix(line, "## ") { + // Metadata. Take the union of annotations across multiple lines, if present. + meta := vendorMeta[mod] + for _, entry := range strings.Split(strings.TrimPrefix(line, "## "), ";") { + entry = strings.TrimSpace(entry) + if entry == "explicit" { + meta.Explicit = true + } + // All other tokens are reserved for future use. + } + vendorMeta[mod] = meta + continue + } + + if f := strings.Fields(line); len(f) == 1 && module.CheckImportPath(f[0]) == nil { + // A package within the current module. + vendorPkgModule[f[0]] = mod + + // Since this module provides a package for the build, we know that it + // is in the build list and is the selected version of its path. + // If this information is new, record it. + if v, ok := vendorVersion[mod.Path]; !ok || semver.Compare(v, mod.Version) < 0 { + vendorList = append(vendorList, mod) + vendorVersion[mod.Path] = mod.Version + } + } + } + }) +} + +// checkVendorConsistency verifies that the vendor/modules.txt file matches (if +// go 1.14) or at least does not contradict (go 1.13 or earlier) the +// requirements and replacements listed in the main module's go.mod file. +func checkVendorConsistency() { + readVendorList() + + pre114 := false + if semver.Compare(index.goVersionV, "v1.14") < 0 { + // Go versions before 1.14 did not include enough information in + // vendor/modules.txt to check for consistency. + // If we know that we're on an earlier version, relax the consistency check. + pre114 = true + } + + vendErrors := new(strings.Builder) + vendErrorf := func(mod module.Version, format string, args ...interface{}) { + detail := fmt.Sprintf(format, args...) + if mod.Version == "" { + fmt.Fprintf(vendErrors, "\n\t%s: %s", mod.Path, detail) + } else { + fmt.Fprintf(vendErrors, "\n\t%s@%s: %s", mod.Path, mod.Version, detail) + } + } + + // Iterate over the Require directives in their original (not indexed) order + // so that the errors match the original file. + for _, r := range modFile.Require { + if !vendorMeta[r.Mod].Explicit { + if pre114 { + // Before 1.14, modules.txt did not indicate whether modules were listed + // explicitly in the main module's go.mod file. + // However, we can at least detect a version mismatch if packages were + // vendored from a non-matching version. + if vv, ok := vendorVersion[r.Mod.Path]; ok && vv != r.Mod.Version { + vendErrorf(r.Mod, fmt.Sprintf("is explicitly required in go.mod, but vendor/modules.txt indicates %s@%s", r.Mod.Path, vv)) + } + } else { + vendErrorf(r.Mod, "is explicitly required in go.mod, but not marked as explicit in vendor/modules.txt") + } + } + } + + describe := func(m module.Version) string { + if m.Version == "" { + return m.Path + } + return m.Path + "@" + m.Version + } + + // We need to verify *all* replacements that occur in modfile: even if they + // don't directly apply to any module in the vendor list, the replacement + // go.mod file can affect the selected versions of other (transitive) + // dependencies + for _, r := range modFile.Replace { + vr := vendorMeta[r.Old].Replacement + if vr == (module.Version{}) { + if pre114 && (r.Old.Version == "" || vendorVersion[r.Old.Path] != r.Old.Version) { + // Before 1.14, modules.txt omitted wildcard replacements and + // replacements for modules that did not have any packages to vendor. + } else { + vendErrorf(r.Old, "is replaced in go.mod, but not marked as replaced in vendor/modules.txt") + } + } else if vr != r.New { + vendErrorf(r.Old, "is replaced by %s in go.mod, but marked as replaced by %s in vendor/modules.txt", describe(r.New), describe(vr)) + } + } + + for _, mod := range vendorList { + meta := vendorMeta[mod] + if meta.Explicit { + if _, inGoMod := index.require[mod]; !inGoMod { + vendErrorf(mod, "is marked as explicit in vendor/modules.txt, but not explicitly required in go.mod") + } + } + } + + for _, mod := range vendorReplaced { + r := Replacement(mod) + if r == (module.Version{}) { + vendErrorf(mod, "is marked as replaced in vendor/modules.txt, but not replaced in go.mod") + continue + } + if meta := vendorMeta[mod]; r != meta.Replacement { + vendErrorf(mod, "is marked as replaced by %s in vendor/modules.txt, but replaced by %s in go.mod", describe(meta.Replacement), describe(r)) + } + } + + if vendErrors.Len() > 0 { + base.Fatalf("go: inconsistent vendoring in %s:%s\n\n\tTo ignore the vendor directory, use -mod=readonly or -mod=mod.\n\tTo sync the vendor directory, run:\n\t\tgo mod vendor", modRoot, vendErrors) + } +} |