diff options
Diffstat (limited to 'src/cmd/go/internal/modload')
-rw-r--r-- | src/cmd/go/internal/modload/build.go | 460 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/buildlist.go | 1510 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/edit.go | 855 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/help.go | 64 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/import.go | 784 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/import_test.go | 97 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/init.go | 2042 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/list.go | 310 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/load.go | 2352 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/modfile.go | 813 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/mvs.go | 136 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/mvs_test.go | 31 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/query.go | 1343 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/query_test.go | 202 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/search.go | 305 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/stat_openfile.go | 28 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/stat_unix.go | 32 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/stat_windows.go | 21 | ||||
-rw-r--r-- | src/cmd/go/internal/modload/vendor.go | 284 |
19 files changed, 11669 insertions, 0 deletions
diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go new file mode 100644 index 0000000..5cf1487 --- /dev/null +++ b/src/cmd/go/internal/modload/build.go @@ -0,0 +1,460 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/gover" + "cmd/go/internal/modfetch" + "cmd/go/internal/modfetch/codehost" + "cmd/go/internal/modindex" + "cmd/go/internal/modinfo" + "cmd/go/internal/search" + + "golang.org/x/mod/module" +) + +var ( + infoStart, _ = hex.DecodeString("3077af0c9274080241e1c107e6d618e6") + infoEnd, _ = hex.DecodeString("f932433186182072008242104116d8f2") +) + +func isStandardImportPath(path string) bool { + return findStandardImportPath(path) != "" +} + +func findStandardImportPath(path string) string { + if path == "" { + panic("findStandardImportPath called with empty path") + } + if search.IsStandardImportPath(path) { + if modindex.IsStandardPackage(cfg.GOROOT, cfg.BuildContext.Compiler, path) { + return filepath.Join(cfg.GOROOT, "src", path) + } + } + return "" +} + +// PackageModuleInfo returns information about the module that provides +// a given package. If modules are not enabled or if the package is in the +// standard library or if the package was not successfully loaded with +// LoadPackages or ImportFromFiles, nil is returned. +func PackageModuleInfo(ctx context.Context, pkgpath string) *modinfo.ModulePublic { + if isStandardImportPath(pkgpath) || !Enabled() { + return nil + } + m, ok := findModule(loaded, pkgpath) + if !ok { + return nil + } + + rs := LoadModFile(ctx) + return moduleInfo(ctx, rs, m, 0, nil) +} + +// PackageModRoot returns the module root directory for the module that provides +// a given package. If modules are not enabled or if the package is in the +// standard library or if the package was not successfully loaded with +// LoadPackages or ImportFromFiles, the empty string is returned. +func PackageModRoot(ctx context.Context, pkgpath string) string { + if isStandardImportPath(pkgpath) || !Enabled() || cfg.BuildMod == "vendor" { + return "" + } + m, ok := findModule(loaded, pkgpath) + if !ok { + return "" + } + root, _, err := fetch(ctx, m) + if err != nil { + return "" + } + return root +} + +func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic { + if !Enabled() { + return nil + } + + if path, vers, found := strings.Cut(path, "@"); found { + m := module.Version{Path: path, Version: vers} + return moduleInfo(ctx, nil, m, 0, nil) + } + + rs := LoadModFile(ctx) + + var ( + v string + ok bool + ) + if rs.pruning == pruned { + v, ok = rs.rootSelected(path) + } + if !ok { + mg, err := rs.Graph(ctx) + if err != nil { + base.Fatal(err) + } + v = mg.Selected(path) + } + + if v == "none" { + return &modinfo.ModulePublic{ + Path: path, + Error: &modinfo.ModuleError{ + Err: "module not in current build", + }, + } + } + + return moduleInfo(ctx, rs, module.Version{Path: path, Version: v}, 0, nil) +} + +// addUpdate fills in m.Update if an updated version is available. +func addUpdate(ctx context.Context, m *modinfo.ModulePublic) { + if m.Version == "" { + return + } + + info, err := Query(ctx, m.Path, "upgrade", m.Version, CheckAllowed) + var noVersionErr *NoMatchingVersionError + if errors.Is(err, ErrDisallowed) || + errors.Is(err, fs.ErrNotExist) || + errors.As(err, &noVersionErr) { + // Ignore "not found" and "no matching version" errors. + // This means the proxy has no matching version or no versions at all. + // + // Ignore "disallowed" errors. This means the current version is + // excluded or retracted and there are no higher allowed versions. + // + // We should report other errors though. An attacker that controls the + // network shouldn't be able to hide versions by interfering with + // the HTTPS connection. An attacker that controls the proxy may still + // hide versions, since the "list" and "latest" endpoints are not + // authenticated. + return + } else if err != nil { + if m.Error == nil { + m.Error = &modinfo.ModuleError{Err: err.Error()} + } + return + } + + if gover.ModCompare(m.Path, info.Version, m.Version) > 0 { + m.Update = &modinfo.ModulePublic{ + Path: m.Path, + Version: info.Version, + Time: &info.Time, + } + } +} + +// mergeOrigin returns the union of data from two origins, +// returning either a new origin or one of its unmodified arguments. +// If the two origins conflict including if either is nil, +// mergeOrigin returns nil. +func mergeOrigin(m1, m2 *codehost.Origin) *codehost.Origin { + if m1 == nil || m2 == nil { + return nil + } + + if m2.VCS != m1.VCS || + m2.URL != m1.URL || + m2.Subdir != m1.Subdir { + return nil + } + + merged := *m1 + if m2.Hash != "" { + if m1.Hash != "" && m1.Hash != m2.Hash { + return nil + } + merged.Hash = m2.Hash + } + if m2.TagSum != "" { + if m1.TagSum != "" && (m1.TagSum != m2.TagSum || m1.TagPrefix != m2.TagPrefix) { + return nil + } + merged.TagSum = m2.TagSum + merged.TagPrefix = m2.TagPrefix + } + if m2.Ref != "" { + if m1.Ref != "" && m1.Ref != m2.Ref { + return nil + } + merged.Ref = m2.Ref + } + + switch { + case merged == *m1: + return m1 + case merged == *m2: + return m2 + default: + // Clone the result to avoid an alloc for merged + // if the result is equal to one of the arguments. + clone := merged + return &clone + } +} + +// addVersions fills in m.Versions with the list of known versions. +// Excluded versions will be omitted. If listRetracted is false, retracted +// versions will also be omitted. +func addVersions(ctx context.Context, m *modinfo.ModulePublic, listRetracted bool) { + // TODO(bcmills): Would it make sense to check for reuse here too? + // Perhaps that doesn't buy us much, though: we would always have to fetch + // all of the version tags to list the available versions anyway. + + allowed := CheckAllowed + if listRetracted { + allowed = CheckExclusions + } + v, origin, err := versions(ctx, m.Path, allowed) + if err != nil && m.Error == nil { + m.Error = &modinfo.ModuleError{Err: err.Error()} + } + m.Versions = v + m.Origin = mergeOrigin(m.Origin, origin) +} + +// addRetraction fills in m.Retracted if the module was retracted by its author. +// m.Error is set if there's an error loading retraction information. +func addRetraction(ctx context.Context, m *modinfo.ModulePublic) { + if m.Version == "" { + return + } + + err := CheckRetractions(ctx, module.Version{Path: m.Path, Version: m.Version}) + var noVersionErr *NoMatchingVersionError + var retractErr *ModuleRetractedError + if err == nil || errors.Is(err, fs.ErrNotExist) || errors.As(err, &noVersionErr) { + // Ignore "not found" and "no matching version" errors. + // This means the proxy has no matching version or no versions at all. + // + // We should report other errors though. An attacker that controls the + // network shouldn't be able to hide versions by interfering with + // the HTTPS connection. An attacker that controls the proxy may still + // hide versions, since the "list" and "latest" endpoints are not + // authenticated. + return + } else if errors.As(err, &retractErr) { + if len(retractErr.Rationale) == 0 { + m.Retracted = []string{"retracted by module author"} + } else { + m.Retracted = retractErr.Rationale + } + } else if m.Error == nil { + m.Error = &modinfo.ModuleError{Err: err.Error()} + } +} + +// addDeprecation fills in m.Deprecated if the module was deprecated by its +// author. m.Error is set if there's an error loading deprecation information. +func addDeprecation(ctx context.Context, m *modinfo.ModulePublic) { + deprecation, err := CheckDeprecation(ctx, module.Version{Path: m.Path, Version: m.Version}) + var noVersionErr *NoMatchingVersionError + if errors.Is(err, fs.ErrNotExist) || errors.As(err, &noVersionErr) { + // Ignore "not found" and "no matching version" errors. + // This means the proxy has no matching version or no versions at all. + // + // We should report other errors though. An attacker that controls the + // network shouldn't be able to hide versions by interfering with + // the HTTPS connection. An attacker that controls the proxy may still + // hide versions, since the "list" and "latest" endpoints are not + // authenticated. + return + } + if err != nil { + if m.Error == nil { + m.Error = &modinfo.ModuleError{Err: err.Error()} + } + return + } + m.Deprecated = deprecation +} + +// moduleInfo returns information about module m, loaded from the requirements +// in rs (which may be nil to indicate that m was not loaded from a requirement +// graph). +func moduleInfo(ctx context.Context, rs *Requirements, m module.Version, mode ListMode, reuse map[module.Version]*modinfo.ModulePublic) *modinfo.ModulePublic { + if m.Version == "" && MainModules.Contains(m.Path) { + info := &modinfo.ModulePublic{ + Path: m.Path, + Version: m.Version, + Main: true, + } + if v, ok := rawGoVersion.Load(m); ok { + info.GoVersion = v.(string) + } else { + panic("internal error: GoVersion not set for main module") + } + if modRoot := MainModules.ModRoot(m); modRoot != "" { + info.Dir = modRoot + info.GoMod = modFilePath(modRoot) + } + return info + } + + info := &modinfo.ModulePublic{ + Path: m.Path, + Version: m.Version, + Indirect: rs != nil && !rs.direct[m.Path], + } + if v, ok := rawGoVersion.Load(m); ok { + info.GoVersion = v.(string) + } + + // completeFromModCache fills in the extra fields in m using the module cache. + completeFromModCache := func(m *modinfo.ModulePublic) { + if gover.IsToolchain(m.Path) { + return + } + + checksumOk := func(suffix string) bool { + return rs == nil || m.Version == "" || !mustHaveSums() || + modfetch.HaveSum(module.Version{Path: m.Path, Version: m.Version + suffix}) + } + + mod := module.Version{Path: m.Path, Version: m.Version} + + if m.Version != "" { + if old := reuse[mod]; old != nil { + if err := checkReuse(ctx, mod, old.Origin); err == nil { + *m = *old + m.Query = "" + m.Dir = "" + return + } + } + + if q, err := Query(ctx, m.Path, m.Version, "", nil); err != nil { + m.Error = &modinfo.ModuleError{Err: err.Error()} + } else { + m.Version = q.Version + m.Time = &q.Time + } + } + + if m.GoVersion == "" && checksumOk("/go.mod") { + // Load the go.mod file to determine the Go version, since it hasn't + // already been populated from rawGoVersion. + if summary, err := rawGoModSummary(mod); err == nil && summary.goVersion != "" { + m.GoVersion = summary.goVersion + } + } + + if m.Version != "" { + if checksumOk("/go.mod") { + gomod, err := modfetch.CachePath(ctx, mod, "mod") + if err == nil { + if info, err := os.Stat(gomod); err == nil && info.Mode().IsRegular() { + m.GoMod = gomod + } + } + } + if checksumOk("") { + dir, err := modfetch.DownloadDir(ctx, mod) + if err == nil { + m.Dir = dir + } + } + + if mode&ListRetracted != 0 { + addRetraction(ctx, m) + } + } + } + + if rs == nil { + // If this was an explicitly-versioned argument to 'go mod download' or + // 'go list -m', report the actual requested version, not its replacement. + completeFromModCache(info) // Will set m.Error in vendor mode. + return info + } + + r := Replacement(m) + if r.Path == "" { + if cfg.BuildMod == "vendor" { + // It's tempting to fill in the "Dir" field to point within the vendor + // directory, but that would be misleading: the vendor directory contains + // a flattened package tree, not complete modules, and it can even + // interleave packages from different modules if one module path is a + // prefix of the other. + } else { + completeFromModCache(info) + } + return info + } + + // Don't hit the network to fill in extra data for replaced modules. + // The original resolved Version and Time don't matter enough to be + // worth the cost, and we're going to overwrite the GoMod and Dir from the + // replacement anyway. See https://golang.org/issue/27859. + info.Replace = &modinfo.ModulePublic{ + Path: r.Path, + Version: r.Version, + } + if v, ok := rawGoVersion.Load(m); ok { + info.Replace.GoVersion = v.(string) + } + if r.Version == "" { + if filepath.IsAbs(r.Path) { + info.Replace.Dir = r.Path + } else { + info.Replace.Dir = filepath.Join(replaceRelativeTo(), r.Path) + } + info.Replace.GoMod = filepath.Join(info.Replace.Dir, "go.mod") + } + if cfg.BuildMod != "vendor" { + completeFromModCache(info.Replace) + info.Dir = info.Replace.Dir + info.GoMod = info.Replace.GoMod + info.Retracted = info.Replace.Retracted + } + info.GoVersion = info.Replace.GoVersion + return info +} + +// findModule searches for the module that contains the package at path. +// If the package was loaded, its containing module and true are returned. +// Otherwise, module.Version{} and false are returned. +func findModule(ld *loader, path string) (module.Version, bool) { + if pkg, ok := ld.pkgCache.Get(path); ok { + return pkg.mod, pkg.mod != module.Version{} + } + return module.Version{}, false +} + +func ModInfoProg(info string, isgccgo bool) []byte { + // Inject an init function to set runtime.modinfo. + // This is only used for gccgo - with gc we hand the info directly to the linker. + // The init function has the drawback that packages may want to + // look at the module info in their init functions (see issue 29628), + // which won't work. See also issue 30344. + if isgccgo { + return fmt.Appendf(nil, `package main +import _ "unsafe" +//go:linkname __set_debug_modinfo__ runtime.setmodinfo +func __set_debug_modinfo__(string) +func init() { __set_debug_modinfo__(%q) } +`, ModInfoData(info)) + } + return nil +} + +func ModInfoData(info string) []byte { + return []byte(string(infoStart) + info + string(infoEnd)) +} diff --git a/src/cmd/go/internal/modload/buildlist.go b/src/cmd/go/internal/modload/buildlist.go new file mode 100644 index 0000000..d72a24f --- /dev/null +++ b/src/cmd/go/internal/modload/buildlist.go @@ -0,0 +1,1510 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "context" + "errors" + "fmt" + "os" + "reflect" + "runtime" + "runtime/debug" + "slices" + "strings" + "sync" + "sync/atomic" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/gover" + "cmd/go/internal/mvs" + "cmd/go/internal/par" + + "golang.org/x/mod/module" +) + +// A Requirements represents a logically-immutable set of root module requirements. +type Requirements struct { + // pruning is the pruning at which the requirement graph is computed. + // + // If unpruned, the graph includes all transitive requirements regardless + // of whether the requiring module supports pruning. + // + // If pruned, the graph includes only the root modules, the explicit + // requirements of those root modules, and the transitive requirements of only + // the root modules that do not support pruning. + // + // If workspace, the graph includes only the workspace modules, the explicit + // requirements of the workspace modules, and the transitive requirements of + // the workspace modules that do not support pruning. + pruning modPruning + + // rootModules is the set of root modules of the graph, sorted and capped to + // length. It may contain duplicates, and may contain multiple versions for a + // given module path. The root modules of the graph are the set of main + // modules in workspace mode, and the main module's direct requirements + // outside workspace mode. + // + // The roots are always expected to contain an entry for the "go" module, + // indicating the Go language version in use. + rootModules []module.Version + maxRootVersion map[string]string + + // direct is the set of module paths for which we believe the module provides + // a package directly imported by a package or test in the main module. + // + // The "direct" map controls which modules are annotated with "// indirect" + // comments in the go.mod file, and may impact which modules are listed as + // explicit roots (vs. indirect-only dependencies). However, it should not + // have a semantic effect on the build list overall. + // + // The initial direct map is populated from the existing "// indirect" + // comments (or lack thereof) in the go.mod file. It is updated by the + // package loader: dependencies may be promoted to direct if new + // direct imports are observed, and may be demoted to indirect during + // 'go mod tidy' or 'go mod vendor'. + // + // The direct map is keyed by module paths, not module versions. When a + // module's selected version changes, we assume that it remains direct if the + // previous version was a direct dependency. That assumption might not hold in + // rare cases (such as if a dependency splits out a nested module, or merges a + // nested module back into a parent module). + direct map[string]bool + + graphOnce sync.Once // guards writes to (but not reads from) graph + graph atomic.Pointer[cachedGraph] +} + +// A cachedGraph is a non-nil *ModuleGraph, together with any error discovered +// while loading that graph. +type cachedGraph struct { + mg *ModuleGraph + err error // If err is non-nil, mg may be incomplete (but must still be non-nil). +} + +// requirements is the requirement graph for the main module. +// +// It is always non-nil if the main module's go.mod file has been loaded. +// +// This variable should only be read from the loadModFile function, and should +// only be written in the loadModFile and commitRequirements functions. +// All other functions that need or produce a *Requirements should +// accept and/or return an explicit parameter. +var requirements *Requirements + +func mustHaveGoRoot(roots []module.Version) { + for _, m := range roots { + if m.Path == "go" { + return + } + } + panic("go: internal error: missing go root module") +} + +// newRequirements returns a new requirement set with the given root modules. +// The dependencies of the roots will be loaded lazily at the first call to the +// Graph method. +// +// The rootModules slice must be sorted according to gover.ModSort. +// The caller must not modify the rootModules slice or direct map after passing +// them to newRequirements. +// +// If vendoring is in effect, the caller must invoke initVendor on the returned +// *Requirements before any other method. +func newRequirements(pruning modPruning, rootModules []module.Version, direct map[string]bool) *Requirements { + mustHaveGoRoot(rootModules) + + if pruning != workspace { + if workFilePath != "" { + panic("in workspace mode, but pruning is not workspace in newRequirements") + } + } + + if pruning != workspace { + if workFilePath != "" { + panic("in workspace mode, but pruning is not workspace in newRequirements") + } + for i, m := range rootModules { + if m.Version == "" && MainModules.Contains(m.Path) { + panic(fmt.Sprintf("newRequirements called with untrimmed build list: rootModules[%v] is a main module", i)) + } + if m.Path == "" || m.Version == "" { + panic(fmt.Sprintf("bad requirement: rootModules[%v] = %v", i, m)) + } + } + } + + rs := &Requirements{ + pruning: pruning, + rootModules: rootModules, + maxRootVersion: make(map[string]string, len(rootModules)), + direct: direct, + } + + for i, m := range rootModules { + if i > 0 { + prev := rootModules[i-1] + if prev.Path > m.Path || (prev.Path == m.Path && gover.ModCompare(m.Path, prev.Version, m.Version) > 0) { + panic(fmt.Sprintf("newRequirements called with unsorted roots: %v", rootModules)) + } + } + + if v, ok := rs.maxRootVersion[m.Path]; ok && gover.ModCompare(m.Path, v, m.Version) >= 0 { + continue + } + rs.maxRootVersion[m.Path] = m.Version + } + + if rs.maxRootVersion["go"] == "" { + panic(`newRequirements called without a "go" version`) + } + return rs +} + +// String returns a string describing the Requirements for debugging. +func (rs *Requirements) String() string { + return fmt.Sprintf("{%v %v}", rs.pruning, rs.rootModules) +} + +// initVendor initializes rs.graph from the given list of vendored module +// dependencies, overriding the graph that would normally be loaded from module +// requirements. +func (rs *Requirements) initVendor(vendorList []module.Version) { + rs.graphOnce.Do(func() { + roots := MainModules.Versions() + if inWorkspaceMode() { + // Use rs.rootModules to pull in the go and toolchain roots + // from the go.work file and preserve the invariant that all + // of rs.rootModules are in mg.g. + roots = rs.rootModules + } + mg := &ModuleGraph{ + g: mvs.NewGraph(cmpVersion, roots), + } + + if rs.pruning == pruned { + mainModule := MainModules.mustGetSingleMainModule() + // The roots of a single pruned module should already include every module in the + // vendor list, because the vendored modules are the same as those needed + // for graph pruning. + // + // Just to be sure, we'll double-check that here. + inconsistent := false + for _, m := range vendorList { + if v, ok := rs.rootSelected(m.Path); !ok || v != m.Version { + base.Errorf("go: vendored module %v should be required explicitly in go.mod", m) + inconsistent = true + } + } + if inconsistent { + base.Fatal(errGoModDirty) + } + + // Now we can treat the rest of the module graph as effectively “pruned + // out”, as though we are viewing the main module from outside: in vendor + // mode, the root requirements *are* the complete module graph. + mg.g.Require(mainModule, rs.rootModules) + } else { + // The transitive requirements of the main module are not in general available + // from the vendor directory, and we don't actually know how we got from + // the roots to the final build list. + // + // Instead, we'll inject a fake "vendor/modules.txt" module that provides + // those transitive dependencies, and mark it as a dependency of the main + // module. That allows us to elide the actual structure of the module + // graph, but still distinguishes between direct and indirect + // dependencies. + vendorMod := module.Version{Path: "vendor/modules.txt", Version: ""} + if inWorkspaceMode() { + for _, m := range MainModules.Versions() { + reqs, _ := rootsFromModFile(m, MainModules.ModFile(m), omitToolchainRoot) + mg.g.Require(m, append(reqs, vendorMod)) + } + mg.g.Require(vendorMod, vendorList) + + } else { + mainModule := MainModules.mustGetSingleMainModule() + mg.g.Require(mainModule, append(rs.rootModules, vendorMod)) + mg.g.Require(vendorMod, vendorList) + } + } + + rs.graph.Store(&cachedGraph{mg, nil}) + }) +} + +// GoVersion returns the Go language version for the Requirements. +func (rs *Requirements) GoVersion() string { + v, _ := rs.rootSelected("go") + if v == "" { + panic("internal error: missing go version in modload.Requirements") + } + return v +} + +// rootSelected returns the version of the root dependency with the given module +// path, or the zero module.Version and ok=false if the module is not a root +// dependency. +func (rs *Requirements) rootSelected(path string) (version string, ok bool) { + if MainModules.Contains(path) { + return "", true + } + if v, ok := rs.maxRootVersion[path]; ok { + return v, true + } + return "", false +} + +// hasRedundantRoot returns true if the root list contains multiple requirements +// of the same module or a requirement on any version of the main module. +// Redundant requirements should be pruned, but they may influence version +// selection. +func (rs *Requirements) hasRedundantRoot() bool { + for i, m := range rs.rootModules { + if MainModules.Contains(m.Path) || (i > 0 && m.Path == rs.rootModules[i-1].Path) { + return true + } + } + return false +} + +// Graph returns the graph of module requirements loaded from the current +// root modules (as reported by RootModules). +// +// Graph always makes a best effort to load the requirement graph despite any +// errors, and always returns a non-nil *ModuleGraph. +// +// If the requirements of any relevant module fail to load, Graph also +// returns a non-nil error of type *mvs.BuildListError. +func (rs *Requirements) Graph(ctx context.Context) (*ModuleGraph, error) { + rs.graphOnce.Do(func() { + mg, mgErr := readModGraph(ctx, rs.pruning, rs.rootModules, nil) + rs.graph.Store(&cachedGraph{mg, mgErr}) + }) + cached := rs.graph.Load() + return cached.mg, cached.err +} + +// IsDirect returns whether the given module provides a package directly +// imported by a package or test in the main module. +func (rs *Requirements) IsDirect(path string) bool { + return rs.direct[path] +} + +// A ModuleGraph represents the complete graph of module dependencies +// of a main module. +// +// If the main module supports module graph pruning, the graph does not include +// transitive dependencies of non-root (implicit) dependencies. +type ModuleGraph struct { + g *mvs.Graph + loadCache par.ErrCache[module.Version, *modFileSummary] + + buildListOnce sync.Once + buildList []module.Version +} + +var readModGraphDebugOnce sync.Once + +// readModGraph reads and returns the module dependency graph starting at the +// given roots. +// +// The requirements of the module versions found in the unprune map are included +// in the graph even if they would normally be pruned out. +// +// Unlike LoadModGraph, readModGraph does not attempt to diagnose or update +// inconsistent roots. +func readModGraph(ctx context.Context, pruning modPruning, roots []module.Version, unprune map[module.Version]bool) (*ModuleGraph, error) { + mustHaveGoRoot(roots) + if pruning == pruned { + // Enable diagnostics for lazy module loading + // (https://golang.org/ref/mod#lazy-loading) only if the module graph is + // pruned. + // + // In unpruned modules,we load the module graph much more aggressively (in + // order to detect inconsistencies that wouldn't be feasible to spot-check), + // so it wouldn't be useful to log when that occurs (because it happens in + // normal operation all the time). + readModGraphDebugOnce.Do(func() { + for _, f := range strings.Split(os.Getenv("GODEBUG"), ",") { + switch f { + case "lazymod=log": + debug.PrintStack() + fmt.Fprintf(os.Stderr, "go: read full module graph.\n") + case "lazymod=strict": + debug.PrintStack() + base.Fatalf("go: read full module graph (forbidden by GODEBUG=lazymod=strict).") + } + } + }) + } + + var graphRoots []module.Version + if inWorkspaceMode() { + graphRoots = roots + } else { + graphRoots = MainModules.Versions() + } + var ( + mu sync.Mutex // guards mg.g and hasError during loading + hasError bool + mg = &ModuleGraph{ + g: mvs.NewGraph(cmpVersion, graphRoots), + } + ) + + if pruning != workspace { + if inWorkspaceMode() { + panic("pruning is not workspace in workspace mode") + } + mg.g.Require(MainModules.mustGetSingleMainModule(), roots) + } + + type dedupKey struct { + m module.Version + pruning modPruning + } + var ( + loadQueue = par.NewQueue(runtime.GOMAXPROCS(0)) + loading sync.Map // dedupKey → nil; the set of modules that have been or are being loaded + ) + + // loadOne synchronously loads the explicit requirements for module m. + // It does not load the transitive requirements of m even if the go version in + // m's go.mod file indicates that it supports graph pruning. + loadOne := func(m module.Version) (*modFileSummary, error) { + return mg.loadCache.Do(m, func() (*modFileSummary, error) { + summary, err := goModSummary(m) + + mu.Lock() + if err == nil { + mg.g.Require(m, summary.require) + } else { + hasError = true + } + mu.Unlock() + + return summary, err + }) + } + + var enqueue func(m module.Version, pruning modPruning) + enqueue = func(m module.Version, pruning modPruning) { + if m.Version == "none" { + return + } + + if _, dup := loading.LoadOrStore(dedupKey{m, pruning}, nil); dup { + // m has already been enqueued for loading. Since unpruned loading may + // follow cycles in the requirement graph, we need to return early + // to avoid making the load queue infinitely long. + return + } + + loadQueue.Add(func() { + summary, err := loadOne(m) + if err != nil { + return // findError will report the error later. + } + + // If the version in m's go.mod file does not support pruning, then we + // cannot assume that the explicit requirements of m (added by loadOne) + // are sufficient to build the packages it contains. We must load its full + // transitive dependency graph to be sure that we see all relevant + // dependencies. In addition, we must load the requirements of any module + // that is explicitly marked as unpruned. + nextPruning := summary.pruning + if pruning == unpruned { + nextPruning = unpruned + } + for _, r := range summary.require { + if pruning != pruned || summary.pruning == unpruned || unprune[r] { + enqueue(r, nextPruning) + } + } + }) + } + + mustHaveGoRoot(roots) + for _, m := range roots { + enqueue(m, pruning) + } + <-loadQueue.Idle() + + // Reload any dependencies of the main modules which are not + // at their selected versions at workspace mode, because the + // requirements don't accurately reflect the transitive imports. + if pruning == workspace { + // hasDepsInAll contains the set of modules that need to be loaded + // at workspace pruning because any of their dependencies may + // provide packages in all. + hasDepsInAll := make(map[string]bool) + seen := map[module.Version]bool{} + for _, m := range roots { + hasDepsInAll[m.Path] = true + } + // This loop will terminate because it will call enqueue on each version of + // each dependency of the modules in hasDepsInAll at most once (and only + // calls enqueue on successively increasing versions of each dependency). + for { + needsEnqueueing := map[module.Version]bool{} + for p := range hasDepsInAll { + m := module.Version{Path: p, Version: mg.g.Selected(p)} + if !seen[m] { + needsEnqueueing[m] = true + continue + } + reqs, _ := mg.g.RequiredBy(m) + for _, r := range reqs { + s := module.Version{Path: r.Path, Version: mg.g.Selected(r.Path)} + if gover.ModCompare(r.Path, s.Version, r.Version) > 0 && !seen[s] { + needsEnqueueing[s] = true + } + } + } + // add all needs enqueueing to paths we care about + if len(needsEnqueueing) == 0 { + break + } + + for p := range needsEnqueueing { + enqueue(p, workspace) + seen[p] = true + hasDepsInAll[p.Path] = true + } + <-loadQueue.Idle() + } + } + + if hasError { + return mg, mg.findError() + } + return mg, nil +} + +// RequiredBy returns the dependencies required by module m in the graph, +// or ok=false if module m's dependencies are pruned out. +// +// The caller must not modify the returned slice, but may safely append to it +// and may rely on it not to be modified. +func (mg *ModuleGraph) RequiredBy(m module.Version) (reqs []module.Version, ok bool) { + return mg.g.RequiredBy(m) +} + +// Selected returns the selected version of the module with the given path. +// +// If no version is selected, Selected returns version "none". +func (mg *ModuleGraph) Selected(path string) (version string) { + return mg.g.Selected(path) +} + +// WalkBreadthFirst invokes f once, in breadth-first order, for each module +// version other than "none" that appears in the graph, regardless of whether +// that version is selected. +func (mg *ModuleGraph) WalkBreadthFirst(f func(m module.Version)) { + mg.g.WalkBreadthFirst(f) +} + +// BuildList returns the selected versions of all modules present in the graph, +// beginning with the main modules. +// +// The order of the remaining elements in the list is deterministic +// but arbitrary. +// +// The caller must not modify the returned list, but may safely append to it +// and may rely on it not to be modified. +func (mg *ModuleGraph) BuildList() []module.Version { + mg.buildListOnce.Do(func() { + mg.buildList = slices.Clip(mg.g.BuildList()) + }) + return mg.buildList +} + +func (mg *ModuleGraph) findError() error { + errStack := mg.g.FindPath(func(m module.Version) bool { + _, err := mg.loadCache.Get(m) + return err != nil && err != par.ErrCacheEntryNotFound + }) + if len(errStack) > 0 { + _, err := mg.loadCache.Get(errStack[len(errStack)-1]) + var noUpgrade func(from, to module.Version) bool + return mvs.NewBuildListError(err, errStack, noUpgrade) + } + + return nil +} + +func (mg *ModuleGraph) allRootsSelected() bool { + var roots []module.Version + if inWorkspaceMode() { + roots = MainModules.Versions() + } else { + roots, _ = mg.g.RequiredBy(MainModules.mustGetSingleMainModule()) + } + for _, m := range roots { + if mg.Selected(m.Path) != m.Version { + return false + } + } + return true +} + +// LoadModGraph loads and returns the graph of module dependencies of the main module, +// without loading any packages. +// +// If the goVersion string is non-empty, the returned graph is the graph +// as interpreted by the given Go version (instead of the version indicated +// in the go.mod file). +// +// Modules are loaded automatically (and lazily) in LoadPackages: +// LoadModGraph need only be called if LoadPackages is not, +// typically in commands that care about modules but no particular package. +func LoadModGraph(ctx context.Context, goVersion string) (*ModuleGraph, error) { + rs, err := loadModFile(ctx, nil) + if err != nil { + return nil, err + } + + if goVersion != "" { + v, _ := rs.rootSelected("go") + if gover.Compare(v, gover.GoStrictVersion) >= 0 && gover.Compare(goVersion, v) < 0 { + return nil, fmt.Errorf("requested Go version %s cannot load module graph (requires Go >= %s)", goVersion, v) + } + + pruning := pruningForGoVersion(goVersion) + if pruning == unpruned && rs.pruning != unpruned { + // Use newRequirements instead of convertDepth because convertDepth + // also updates roots; here, we want to report the unmodified roots + // even though they may seem inconsistent. + rs = newRequirements(unpruned, rs.rootModules, rs.direct) + } + + return rs.Graph(ctx) + } + + rs, mg, err := expandGraph(ctx, rs) + if err != nil { + return nil, err + } + requirements = rs + return mg, err +} + +// expandGraph loads the complete module graph from rs. +// +// If the complete graph reveals that some root of rs is not actually the +// selected version of its path, expandGraph computes a new set of roots that +// are consistent. (With a pruned module graph, this may result in upgrades to +// other modules due to requirements that were previously pruned out.) +// +// expandGraph returns the updated roots, along with the module graph loaded +// from those roots and any error encountered while loading that graph. +// expandGraph returns non-nil requirements and a non-nil graph regardless of +// errors. On error, the roots might not be updated to be consistent. +func expandGraph(ctx context.Context, rs *Requirements) (*Requirements, *ModuleGraph, error) { + mg, mgErr := rs.Graph(ctx) + if mgErr != nil { + // Without the graph, we can't update the roots: we don't know which + // versions of transitive dependencies would be selected. + return rs, mg, mgErr + } + + if !mg.allRootsSelected() { + // The roots of rs are not consistent with the rest of the graph. Update + // them. In an unpruned module this is a no-op for the build list as a whole — + // it just promotes what were previously transitive requirements to be + // roots — but in a pruned module it may pull in previously-irrelevant + // transitive dependencies. + + newRS, rsErr := updateRoots(ctx, rs.direct, rs, nil, nil, false) + if rsErr != nil { + // Failed to update roots, perhaps because of an error in a transitive + // dependency needed for the update. Return the original Requirements + // instead. + return rs, mg, rsErr + } + rs = newRS + mg, mgErr = rs.Graph(ctx) + } + + return rs, mg, mgErr +} + +// EditBuildList edits the global build list by first adding every module in add +// to the existing build list, then adjusting versions (and adding or removing +// requirements as needed) until every module in mustSelect is selected at the +// given version. +// +// (Note that the newly-added modules might not be selected in the resulting +// build list: they could be lower than existing requirements or conflict with +// versions in mustSelect.) +// +// If the versions listed in mustSelect are mutually incompatible (due to one of +// the listed modules requiring a higher version of another), EditBuildList +// returns a *ConstraintError and leaves the build list in its previous state. +// +// On success, EditBuildList reports whether the selected version of any module +// in the build list may have been changed (possibly to or from "none") as a +// result. +func EditBuildList(ctx context.Context, add, mustSelect []module.Version) (changed bool, err error) { + rs, changed, err := editRequirements(ctx, LoadModFile(ctx), add, mustSelect) + if err != nil { + return false, err + } + requirements = rs + return changed, err +} + +// OverrideRoots edits the global requirement roots by replacing the specific module versions. +func OverrideRoots(ctx context.Context, replace []module.Version) { + requirements = overrideRoots(ctx, requirements, replace) +} + +func overrideRoots(ctx context.Context, rs *Requirements, replace []module.Version) *Requirements { + drop := make(map[string]bool) + for _, m := range replace { + drop[m.Path] = true + } + var roots []module.Version + for _, m := range rs.rootModules { + if !drop[m.Path] { + roots = append(roots, m) + } + } + roots = append(roots, replace...) + gover.ModSort(roots) + return newRequirements(rs.pruning, roots, rs.direct) +} + +// A ConstraintError describes inconsistent constraints in EditBuildList +type ConstraintError struct { + // Conflict lists the source of the conflict for each version in mustSelect + // that could not be selected due to the requirements of some other version in + // mustSelect. + Conflicts []Conflict +} + +func (e *ConstraintError) Error() string { + b := new(strings.Builder) + b.WriteString("version constraints conflict:") + for _, c := range e.Conflicts { + fmt.Fprintf(b, "\n\t%s", c.Summary()) + } + return b.String() +} + +// A Conflict is a path of requirements starting at a root or proposed root in +// the requirement graph, explaining why that root either causes a module passed +// in the mustSelect list to EditBuildList to be unattainable, or introduces an +// unresolvable error in loading the requirement graph. +type Conflict struct { + // Path is a path of requirements starting at some module version passed in + // the mustSelect argument and ending at a module whose requirements make that + // version unacceptable. (Path always has len ≥ 1.) + Path []module.Version + + // If Err is nil, Constraint is a module version passed in the mustSelect + // argument that has the same module path as, and a lower version than, + // the last element of the Path slice. + Constraint module.Version + + // If Constraint is unset, Err is an error encountered when loading the + // requirements of the last element in Path. + Err error +} + +// UnwrapModuleError returns c.Err, but unwraps it if it is a module.ModuleError +// with a version and path matching the last entry in the Path slice. +func (c Conflict) UnwrapModuleError() error { + me, ok := c.Err.(*module.ModuleError) + if ok && len(c.Path) > 0 { + last := c.Path[len(c.Path)-1] + if me.Path == last.Path && me.Version == last.Version { + return me.Err + } + } + return c.Err +} + +// Summary returns a string that describes only the first and last modules in +// the conflict path. +func (c Conflict) Summary() string { + if len(c.Path) == 0 { + return "(internal error: invalid Conflict struct)" + } + first := c.Path[0] + last := c.Path[len(c.Path)-1] + if len(c.Path) == 1 { + if c.Err != nil { + return fmt.Sprintf("%s: %v", first, c.UnwrapModuleError()) + } + return fmt.Sprintf("%s is above %s", first, c.Constraint.Version) + } + + adverb := "" + if len(c.Path) > 2 { + adverb = "indirectly " + } + if c.Err != nil { + return fmt.Sprintf("%s %srequires %s: %v", first, adverb, last, c.UnwrapModuleError()) + } + return fmt.Sprintf("%s %srequires %s, but %s is requested", first, adverb, last, c.Constraint.Version) +} + +// String returns a string that describes the full conflict path. +func (c Conflict) String() string { + if len(c.Path) == 0 { + return "(internal error: invalid Conflict struct)" + } + b := new(strings.Builder) + fmt.Fprintf(b, "%v", c.Path[0]) + if len(c.Path) == 1 { + fmt.Fprintf(b, " found") + } else { + for _, r := range c.Path[1:] { + fmt.Fprintf(b, " requires\n\t%v", r) + } + } + if c.Constraint != (module.Version{}) { + fmt.Fprintf(b, ", but %v is requested", c.Constraint.Version) + } + if c.Err != nil { + fmt.Fprintf(b, ": %v", c.UnwrapModuleError()) + } + return b.String() +} + +// tidyRoots trims the root dependencies to the minimal requirements needed to +// both retain the same versions of all packages in pkgs and satisfy the +// graph-pruning invariants (if applicable). +func tidyRoots(ctx context.Context, rs *Requirements, pkgs []*loadPkg) (*Requirements, error) { + mainModule := MainModules.mustGetSingleMainModule() + if rs.pruning == unpruned { + return tidyUnprunedRoots(ctx, mainModule, rs, pkgs) + } + return tidyPrunedRoots(ctx, mainModule, rs, pkgs) +} + +func updateRoots(ctx context.Context, direct map[string]bool, rs *Requirements, pkgs []*loadPkg, add []module.Version, rootsImported bool) (*Requirements, error) { + switch rs.pruning { + case unpruned: + return updateUnprunedRoots(ctx, direct, rs, add) + case pruned: + return updatePrunedRoots(ctx, direct, rs, pkgs, add, rootsImported) + case workspace: + return updateWorkspaceRoots(ctx, rs, add) + default: + panic(fmt.Sprintf("unsupported pruning mode: %v", rs.pruning)) + } +} + +func updateWorkspaceRoots(ctx context.Context, rs *Requirements, add []module.Version) (*Requirements, error) { + if len(add) != 0 { + // add should be empty in workspace mode because workspace mode implies + // -mod=readonly, which in turn implies no new requirements. The code path + // that would result in add being non-empty returns an error before it + // reaches this point: The set of modules to add comes from + // resolveMissingImports, which in turn resolves each package by calling + // queryImport. But queryImport explicitly checks for -mod=readonly, and + // return an error. + panic("add is not empty") + } + return rs, nil +} + +// tidyPrunedRoots returns a minimal set of root requirements that maintains the +// invariants of the go.mod file needed to support graph pruning for the given +// packages: +// +// 1. For each package marked with pkgInAll, the module path that provided that +// package is included as a root. +// 2. For all packages, the module that provided that package either remains +// selected at the same version or is upgraded by the dependencies of a +// root. +// +// If any module that provided a package has been upgraded above its previous +// version, the caller may need to reload and recompute the package graph. +// +// To ensure that the loading process eventually converges, the caller should +// add any needed roots from the tidy root set (without removing existing untidy +// roots) until the set of roots has converged. +func tidyPrunedRoots(ctx context.Context, mainModule module.Version, old *Requirements, pkgs []*loadPkg) (*Requirements, error) { + var ( + roots []module.Version + pathIsRoot = map[string]bool{mainModule.Path: true} + ) + if v, ok := old.rootSelected("go"); ok { + roots = append(roots, module.Version{Path: "go", Version: v}) + pathIsRoot["go"] = true + } + if v, ok := old.rootSelected("toolchain"); ok { + roots = append(roots, module.Version{Path: "toolchain", Version: v}) + pathIsRoot["toolchain"] = true + } + // We start by adding roots for every package in "all". + // + // Once that is done, we may still need to add more roots to cover upgraded or + // otherwise-missing test dependencies for packages in "all". For those test + // dependencies, we prefer to add roots for packages with shorter import + // stacks first, on the theory that the module requirements for those will + // tend to fill in the requirements for their transitive imports (which have + // deeper import stacks). So we add the missing dependencies for one depth at + // a time, starting with the packages actually in "all" and expanding outwards + // until we have scanned every package that was loaded. + var ( + queue []*loadPkg + queued = map[*loadPkg]bool{} + ) + for _, pkg := range pkgs { + if !pkg.flags.has(pkgInAll) { + continue + } + if pkg.fromExternalModule() && !pathIsRoot[pkg.mod.Path] { + roots = append(roots, pkg.mod) + pathIsRoot[pkg.mod.Path] = true + } + queue = append(queue, pkg) + queued[pkg] = true + } + gover.ModSort(roots) + tidy := newRequirements(pruned, roots, old.direct) + + for len(queue) > 0 { + roots = tidy.rootModules + mg, err := tidy.Graph(ctx) + if err != nil { + return nil, err + } + + prevQueue := queue + queue = nil + for _, pkg := range prevQueue { + m := pkg.mod + if m.Path == "" { + continue + } + for _, dep := range pkg.imports { + if !queued[dep] { + queue = append(queue, dep) + queued[dep] = true + } + } + if pkg.test != nil && !queued[pkg.test] { + queue = append(queue, pkg.test) + queued[pkg.test] = true + } + + if !pathIsRoot[m.Path] { + if s := mg.Selected(m.Path); gover.ModCompare(m.Path, s, m.Version) < 0 { + roots = append(roots, m) + pathIsRoot[m.Path] = true + } + } + } + + if len(roots) > len(tidy.rootModules) { + gover.ModSort(roots) + tidy = newRequirements(pruned, roots, tidy.direct) + } + } + + roots = tidy.rootModules + _, err := tidy.Graph(ctx) + if err != nil { + return nil, err + } + + // We try to avoid adding explicit requirements for test-only dependencies of + // packages in external modules. However, if we drop the explicit + // requirements, that may change an import from unambiguous (due to lazy + // module loading) to ambiguous (because lazy module loading no longer + // disambiguates it). For any package that has become ambiguous, we try + // to fix it by promoting its module to an explicit root. + // (See https://go.dev/issue/60313.) + q := par.NewQueue(runtime.GOMAXPROCS(0)) + for { + var disambiguateRoot sync.Map + for _, pkg := range pkgs { + if pkg.mod.Path == "" || pathIsRoot[pkg.mod.Path] { + // Lazy module loading will cause pkg.mod to be checked before any other modules + // that are only indirectly required. It is as unambiguous as possible. + continue + } + pkg := pkg + q.Add(func() { + skipModFile := true + _, _, _, _, err := importFromModules(ctx, pkg.path, tidy, nil, skipModFile) + if aie := (*AmbiguousImportError)(nil); errors.As(err, &aie) { + disambiguateRoot.Store(pkg.mod, true) + } + }) + } + <-q.Idle() + + disambiguateRoot.Range(func(k, _ any) bool { + m := k.(module.Version) + roots = append(roots, m) + pathIsRoot[m.Path] = true + return true + }) + + if len(roots) > len(tidy.rootModules) { + module.Sort(roots) + tidy = newRequirements(pruned, roots, tidy.direct) + _, err = tidy.Graph(ctx) + if err != nil { + return nil, err + } + // Adding these roots may have pulled additional modules into the module + // graph, causing additional packages to become ambiguous. Keep iterating + // until we reach a fixed point. + continue + } + + break + } + + return tidy, nil +} + +// updatePrunedRoots returns a set of root requirements that maintains the +// invariants of the go.mod file needed to support graph pruning: +// +// 1. The selected version of the module providing each package marked with +// either pkgInAll or pkgIsRoot is included as a root. +// Note that certain root patterns (such as '...') may explode the root set +// to contain every module that provides any package imported (or merely +// required) by any other module. +// 2. Each root appears only once, at the selected version of its path +// (if rs.graph is non-nil) or at the highest version otherwise present as a +// root (otherwise). +// 3. Every module path that appears as a root in rs remains a root. +// 4. Every version in add is selected at its given version unless upgraded by +// (the dependencies of) an existing root or another module in add. +// +// The packages in pkgs are assumed to have been loaded from either the roots of +// rs or the modules selected in the graph of rs. +// +// The above invariants together imply the graph-pruning invariants for the +// go.mod file: +// +// 1. (The import invariant.) Every module that provides a package transitively +// imported by any package or test in the main module is included as a root. +// This follows by induction from (1) and (3) above. Transitively-imported +// packages loaded during this invocation are marked with pkgInAll (1), +// and by hypothesis any transitively-imported packages loaded in previous +// invocations were already roots in rs (3). +// +// 2. (The argument invariant.) Every module that provides a package matching +// an explicit package pattern is included as a root. This follows directly +// from (1): packages matching explicit package patterns are marked with +// pkgIsRoot. +// +// 3. (The completeness invariant.) Every module that contributed any package +// to the build is required by either the main module or one of the modules +// it requires explicitly. This invariant is left up to the caller, who must +// not load packages from outside the module graph but may add roots to the +// graph, but is facilitated by (3). If the caller adds roots to the graph in +// order to resolve missing packages, then updatePrunedRoots will retain them, +// the selected versions of those roots cannot regress, and they will +// eventually be written back to the main module's go.mod file. +// +// (See https://golang.org/design/36460-lazy-module-loading#invariants for more +// detail.) +func updatePrunedRoots(ctx context.Context, direct map[string]bool, rs *Requirements, pkgs []*loadPkg, add []module.Version, rootsImported bool) (*Requirements, error) { + roots := rs.rootModules + rootsUpgraded := false + + spotCheckRoot := map[module.Version]bool{} + + // “The selected version of the module providing each package marked with + // either pkgInAll or pkgIsRoot is included as a root.” + needSort := false + for _, pkg := range pkgs { + if !pkg.fromExternalModule() { + // pkg was not loaded from a module dependency, so we don't need + // to do anything special to maintain that dependency. + continue + } + + switch { + case pkg.flags.has(pkgInAll): + // pkg is transitively imported by a package or test in the main module. + // We need to promote the module that maintains it to a root: if some + // other module depends on the main module, and that other module also + // uses a pruned module graph, it will expect to find all of our + // transitive dependencies by reading just our go.mod file, not the go.mod + // files of everything we depend on. + // + // (This is the “import invariant” that makes graph pruning possible.) + + case rootsImported && pkg.flags.has(pkgFromRoot): + // pkg is a transitive dependency of some root, and we are treating the + // roots as if they are imported by the main module (as in 'go get'). + + case pkg.flags.has(pkgIsRoot): + // pkg is a root of the package-import graph. (Generally this means that + // it matches a command-line argument.) We want future invocations of the + // 'go' command — such as 'go test' on the same package — to continue to + // use the same versions of its dependencies that we are using right now. + // So we need to bring this package's dependencies inside the pruned + // module graph. + // + // Making the module containing this package a root of the module graph + // does exactly that: if the module containing the package supports graph + // pruning then it should satisfy the import invariant itself, so all of + // its dependencies should be in its go.mod file, and if the module + // containing the package does not support pruning then if we make it a + // root we will load all of its (unpruned) transitive dependencies into + // the module graph. + // + // (This is the “argument invariant”, and is important for + // reproducibility.) + + default: + // pkg is a dependency of some other package outside of the main module. + // As far as we know it's not relevant to the main module (and thus not + // relevant to consumers of the main module either), and its dependencies + // should already be in the module graph — included in the dependencies of + // the package that imported it. + continue + } + + if _, ok := rs.rootSelected(pkg.mod.Path); ok { + // It is possible that the main module's go.mod file is incomplete or + // otherwise erroneous — for example, perhaps the author forgot to 'git + // add' their updated go.mod file after adding a new package import, or + // perhaps they made an edit to the go.mod file using a third-party tool + // ('git merge'?) that doesn't maintain consistency for module + // dependencies. If that happens, ideally we want to detect the missing + // requirements and fix them up here. + // + // However, we also need to be careful not to be too aggressive. For + // transitive dependencies of external tests, the go.mod file for the + // module containing the test itself is expected to provide all of the + // relevant dependencies, and we explicitly don't want to pull in + // requirements on *irrelevant* requirements that happen to occur in the + // go.mod files for these transitive-test-only dependencies. (See the test + // in mod_lazy_test_horizon.txt for a concrete example). + // + // The “goldilocks zone” seems to be to spot-check exactly the same + // modules that we promote to explicit roots: namely, those that provide + // packages transitively imported by the main module, and those that + // provide roots of the package-import graph. That will catch erroneous + // edits to the main module's go.mod file and inconsistent requirements in + // dependencies that provide imported packages, but will ignore erroneous + // or misleading requirements in dependencies that aren't obviously + // relevant to the packages in the main module. + spotCheckRoot[pkg.mod] = true + } else { + roots = append(roots, pkg.mod) + rootsUpgraded = true + // The roots slice was initially sorted because rs.rootModules was sorted, + // but the root we just added could be out of order. + needSort = true + } + } + + for _, m := range add { + if v, ok := rs.rootSelected(m.Path); !ok || gover.ModCompare(m.Path, v, m.Version) < 0 { + roots = append(roots, m) + rootsUpgraded = true + needSort = true + } + } + if needSort { + gover.ModSort(roots) + } + + // "Each root appears only once, at the selected version of its path ….” + for { + var mg *ModuleGraph + if rootsUpgraded { + // We've added or upgraded one or more roots, so load the full module + // graph so that we can update those roots to be consistent with other + // requirements. + if mustHaveCompleteRequirements() { + // Our changes to the roots may have moved dependencies into or out of + // the graph-pruning horizon, which could in turn change the selected + // versions of other modules. (For pruned modules adding or removing an + // explicit root is a semantic change, not just a cosmetic one.) + return rs, errGoModDirty + } + + rs = newRequirements(pruned, roots, direct) + var err error + mg, err = rs.Graph(ctx) + if err != nil { + return rs, err + } + } else { + // Since none of the roots have been upgraded, we have no reason to + // suspect that they are inconsistent with the requirements of any other + // roots. Only look at the full module graph if we've already loaded it; + // otherwise, just spot-check the explicit requirements of the roots from + // which we loaded packages. + if rs.graph.Load() != nil { + // We've already loaded the full module graph, which includes the + // requirements of all of the root modules — even the transitive + // requirements, if they are unpruned! + mg, _ = rs.Graph(ctx) + } else if cfg.BuildMod == "vendor" { + // We can't spot-check the requirements of other modules because we + // don't in general have their go.mod files available in the vendor + // directory. (Fortunately this case is impossible, because mg.graph is + // always non-nil in vendor mode!) + panic("internal error: rs.graph is unexpectedly nil with -mod=vendor") + } else if !spotCheckRoots(ctx, rs, spotCheckRoot) { + // We spot-checked the explicit requirements of the roots that are + // relevant to the packages we've loaded. Unfortunately, they're + // inconsistent in some way; we need to load the full module graph + // so that we can fix the roots properly. + var err error + mg, err = rs.Graph(ctx) + if err != nil { + return rs, err + } + } + } + + roots = make([]module.Version, 0, len(rs.rootModules)) + rootsUpgraded = false + inRootPaths := make(map[string]bool, len(rs.rootModules)+1) + for _, mm := range MainModules.Versions() { + inRootPaths[mm.Path] = true + } + for _, m := range rs.rootModules { + if inRootPaths[m.Path] { + // This root specifies a redundant path. We already retained the + // selected version of this path when we saw it before, so omit the + // redundant copy regardless of its version. + // + // When we read the full module graph, we include the dependencies of + // every root even if that root is redundant. That better preserves + // reproducibility if, say, some automated tool adds a redundant + // 'require' line and then runs 'go mod tidy' to try to make everything + // consistent, since the requirements of the older version are carried + // over. + // + // So omitting a root that was previously present may *reduce* the + // selected versions of non-roots, but merely removing a requirement + // cannot *increase* the selected versions of other roots as a result — + // we don't need to mark this change as an upgrade. (This particular + // change cannot invalidate any other roots.) + continue + } + + var v string + if mg == nil { + v, _ = rs.rootSelected(m.Path) + } else { + v = mg.Selected(m.Path) + } + roots = append(roots, module.Version{Path: m.Path, Version: v}) + inRootPaths[m.Path] = true + if v != m.Version { + rootsUpgraded = true + } + } + // Note that rs.rootModules was already sorted by module path and version, + // and we appended to the roots slice in the same order and guaranteed that + // each path has only one version, so roots is also sorted by module path + // and (trivially) version. + + if !rootsUpgraded { + if cfg.BuildMod != "mod" { + // The only changes to the root set (if any) were to remove duplicates. + // The requirements are consistent (if perhaps redundant), so keep the + // original rs to preserve its ModuleGraph. + return rs, nil + } + // The root set has converged: every root going into this iteration was + // already at its selected version, although we have have removed other + // (redundant) roots for the same path. + break + } + } + + if rs.pruning == pruned && reflect.DeepEqual(roots, rs.rootModules) && reflect.DeepEqual(direct, rs.direct) { + // The root set is unchanged and rs was already pruned, so keep rs to + // preserve its cached ModuleGraph (if any). + return rs, nil + } + return newRequirements(pruned, roots, direct), nil +} + +// spotCheckRoots reports whether the versions of the roots in rs satisfy the +// explicit requirements of the modules in mods. +func spotCheckRoots(ctx context.Context, rs *Requirements, mods map[module.Version]bool) bool { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + work := par.NewQueue(runtime.GOMAXPROCS(0)) + for m := range mods { + m := m + work.Add(func() { + if ctx.Err() != nil { + return + } + + summary, err := goModSummary(m) + if err != nil { + cancel() + return + } + + for _, r := range summary.require { + if v, ok := rs.rootSelected(r.Path); ok && gover.ModCompare(r.Path, v, r.Version) < 0 { + cancel() + return + } + } + }) + } + <-work.Idle() + + if ctx.Err() != nil { + // Either we failed a spot-check, or the caller no longer cares about our + // answer anyway. + return false + } + + return true +} + +// tidyUnprunedRoots returns a minimal set of root requirements that maintains +// the selected version of every module that provided or lexically could have +// provided a package in pkgs, and includes the selected version of every such +// module in direct as a root. +func tidyUnprunedRoots(ctx context.Context, mainModule module.Version, old *Requirements, pkgs []*loadPkg) (*Requirements, error) { + var ( + // keep is a set of of modules that provide packages or are needed to + // disambiguate imports. + keep []module.Version + keptPath = map[string]bool{} + + // rootPaths is a list of module paths that provide packages directly + // imported from the main module. They should be included as roots. + rootPaths []string + inRootPaths = map[string]bool{} + + // altMods is a set of paths of modules that lexically could have provided + // imported packages. It may be okay to remove these from the list of + // explicit requirements if that removes them from the module graph. If they + // are present in the module graph reachable from rootPaths, they must not + // be at a lower version. That could cause a missing sum error or a new + // import ambiguity. + // + // For example, suppose a developer rewrites imports from example.com/m to + // example.com/m/v2, then runs 'go mod tidy'. Tidy may delete the + // requirement on example.com/m if there is no other transitive requirement + // on it. However, if example.com/m were downgraded to a version not in + // go.sum, when package example.com/m/v2/p is loaded, we'd get an error + // trying to disambiguate the import, since we can't check example.com/m + // without its sum. See #47738. + altMods = map[string]string{} + ) + if v, ok := old.rootSelected("go"); ok { + keep = append(keep, module.Version{Path: "go", Version: v}) + keptPath["go"] = true + } + if v, ok := old.rootSelected("toolchain"); ok { + keep = append(keep, module.Version{Path: "toolchain", Version: v}) + keptPath["toolchain"] = true + } + for _, pkg := range pkgs { + if !pkg.fromExternalModule() { + continue + } + if m := pkg.mod; !keptPath[m.Path] { + keep = append(keep, m) + keptPath[m.Path] = true + if old.direct[m.Path] && !inRootPaths[m.Path] { + rootPaths = append(rootPaths, m.Path) + inRootPaths[m.Path] = true + } + } + for _, m := range pkg.altMods { + altMods[m.Path] = m.Version + } + } + + // Construct a build list with a minimal set of roots. + // This may remove or downgrade modules in altMods. + reqs := &mvsReqs{roots: keep} + min, err := mvs.Req(mainModule, rootPaths, reqs) + if err != nil { + return nil, err + } + buildList, err := mvs.BuildList([]module.Version{mainModule}, reqs) + if err != nil { + return nil, err + } + + // Check if modules in altMods were downgraded but not removed. + // If so, add them to roots, which will retain an "// indirect" requirement + // in go.mod. See comment on altMods above. + keptAltMod := false + for _, m := range buildList { + if v, ok := altMods[m.Path]; ok && gover.ModCompare(m.Path, m.Version, v) < 0 { + keep = append(keep, module.Version{Path: m.Path, Version: v}) + keptAltMod = true + } + } + if keptAltMod { + // We must run mvs.Req again instead of simply adding altMods to min. + // It's possible that a requirement in altMods makes some other + // explicit indirect requirement unnecessary. + reqs.roots = keep + min, err = mvs.Req(mainModule, rootPaths, reqs) + if err != nil { + return nil, err + } + } + + return newRequirements(unpruned, min, old.direct), nil +} + +// updateUnprunedRoots returns a set of root requirements that includes the selected +// version of every module path in direct as a root, and maintains the selected +// version of every module selected in the graph of rs. +// +// The roots are updated such that: +// +// 1. The selected version of every module path in direct is included as a root +// (if it is not "none"). +// 2. Each root is the selected version of its path. (We say that such a root +// set is “consistent”.) +// 3. Every version selected in the graph of rs remains selected unless upgraded +// by a dependency in add. +// 4. Every version in add is selected at its given version unless upgraded by +// (the dependencies of) an existing root or another module in add. +func updateUnprunedRoots(ctx context.Context, direct map[string]bool, rs *Requirements, add []module.Version) (*Requirements, error) { + mg, err := rs.Graph(ctx) + if err != nil { + // We can't ignore errors in the module graph even if the user passed the -e + // flag to try to push past them. If we can't load the complete module + // dependencies, then we can't reliably compute a minimal subset of them. + return rs, err + } + + if mustHaveCompleteRequirements() { + // Instead of actually updating the requirements, just check that no updates + // are needed. + if rs == nil { + // We're being asked to reconstruct the requirements from scratch, + // but we aren't even allowed to modify them. + return rs, errGoModDirty + } + for _, m := range rs.rootModules { + if m.Version != mg.Selected(m.Path) { + // The root version v is misleading: the actual selected version is higher. + return rs, errGoModDirty + } + } + for _, m := range add { + if m.Version != mg.Selected(m.Path) { + return rs, errGoModDirty + } + } + for mPath := range direct { + if _, ok := rs.rootSelected(mPath); !ok { + // Module m is supposed to be listed explicitly, but isn't. + // + // Note that this condition is also detected (and logged with more + // detail) earlier during package loading, so it shouldn't actually be + // possible at this point — this is just a defense in depth. + return rs, errGoModDirty + } + } + + // No explicit roots are missing and all roots are already at the versions + // we want to keep. Any other changes we would make are purely cosmetic, + // such as pruning redundant indirect dependencies. Per issue #34822, we + // ignore cosmetic changes when we cannot update the go.mod file. + return rs, nil + } + + var ( + rootPaths []string // module paths that should be included as roots + inRootPaths = map[string]bool{} + ) + for _, root := range rs.rootModules { + // If the selected version of the root is the same as what was already + // listed in the go.mod file, retain it as a root (even if redundant) to + // avoid unnecessary churn. (See https://golang.org/issue/34822.) + // + // We do this even for indirect requirements, since we don't know why they + // were added and they could become direct at any time. + if !inRootPaths[root.Path] && mg.Selected(root.Path) == root.Version { + rootPaths = append(rootPaths, root.Path) + inRootPaths[root.Path] = true + } + } + + // “The selected version of every module path in direct is included as a root.” + // + // This is only for convenience and clarity for end users: in an unpruned module, + // the choice of explicit vs. implicit dependency has no impact on MVS + // selection (for itself or any other module). + keep := append(mg.BuildList()[MainModules.Len():], add...) + for _, m := range keep { + if direct[m.Path] && !inRootPaths[m.Path] { + rootPaths = append(rootPaths, m.Path) + inRootPaths[m.Path] = true + } + } + + var roots []module.Version + for _, mainModule := range MainModules.Versions() { + min, err := mvs.Req(mainModule, rootPaths, &mvsReqs{roots: keep}) + if err != nil { + return rs, err + } + roots = append(roots, min...) + } + if MainModules.Len() > 1 { + gover.ModSort(roots) + } + if rs.pruning == unpruned && reflect.DeepEqual(roots, rs.rootModules) && reflect.DeepEqual(direct, rs.direct) { + // The root set is unchanged and rs was already unpruned, so keep rs to + // preserve its cached ModuleGraph (if any). + return rs, nil + } + + return newRequirements(unpruned, roots, direct), nil +} + +// convertPruning returns a version of rs with the given pruning behavior. +// If rs already has the given pruning, convertPruning returns rs unmodified. +func convertPruning(ctx context.Context, rs *Requirements, pruning modPruning) (*Requirements, error) { + if rs.pruning == pruning { + return rs, nil + } else if rs.pruning == workspace || pruning == workspace { + panic("attempting to convert to/from workspace pruning and another pruning type") + } + + if pruning == unpruned { + // We are converting a pruned module to an unpruned one. The roots of a + // pruned module graph are a superset of the roots of an unpruned one, so + // we don't need to add any new roots — we just need to drop the ones that + // are redundant, which is exactly what updateUnprunedRoots does. + return updateUnprunedRoots(ctx, rs.direct, rs, nil) + } + + // We are converting an unpruned module to a pruned one. + // + // An unpruned module graph includes the transitive dependencies of every + // module in the build list. As it turns out, we can express that as a pruned + // root set! “Include the transitive dependencies of every module in the build + // list” is exactly what happens in a pruned module if we promote every module + // in the build list to a root. + mg, err := rs.Graph(ctx) + if err != nil { + return rs, err + } + return newRequirements(pruned, mg.BuildList()[MainModules.Len():], rs.direct), nil +} diff --git a/src/cmd/go/internal/modload/edit.go b/src/cmd/go/internal/modload/edit.go new file mode 100644 index 0000000..63ee15c --- /dev/null +++ b/src/cmd/go/internal/modload/edit.go @@ -0,0 +1,855 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "cmd/go/internal/cfg" + "cmd/go/internal/gover" + "cmd/go/internal/mvs" + "cmd/go/internal/par" + "context" + "errors" + "fmt" + "maps" + "os" + "slices" + + "golang.org/x/mod/module" +) + +// editRequirements returns an edited version of rs such that: +// +// 1. Each module version in mustSelect is selected. +// +// 2. Each module version in tryUpgrade is upgraded toward the indicated +// version as far as can be done without violating (1). +// (Other upgrades are also allowed if they are caused by +// transitive requirements of versions in mustSelect or +// tryUpgrade.) +// +// 3. Each module version in rs.rootModules (or rs.graph, if rs is unpruned) +// is downgraded or upgraded from its original version only to the extent +// needed to satisfy (1) and (2). +// +// Generally, the module versions in mustSelect are due to the module or a +// package within the module matching an explicit command line argument to 'go +// get', and the versions in tryUpgrade are transitive dependencies that are +// either being upgraded by 'go get -u' or being added to satisfy some +// otherwise-missing package import. +// +// If pruning is enabled, the roots of the edited requirements include an +// explicit entry for each module path in tryUpgrade, mustSelect, and the roots +// of rs, unless the selected version for the module path is "none". +func editRequirements(ctx context.Context, rs *Requirements, tryUpgrade, mustSelect []module.Version) (edited *Requirements, changed bool, err error) { + if rs.pruning == workspace { + panic("editRequirements cannot edit workspace requirements") + } + + orig := rs + // If we already know what go version we will end up on after the edit, and + // the pruning for that version is different, go ahead and apply it now. + // + // If we are changing from pruned to unpruned, then we MUST check the unpruned + // graph for conflicts from the start. (Checking only for pruned conflicts + // would miss some that would be introduced later.) + // + // If we are changing from unpruned to pruned, then we would like to avoid + // unnecessary downgrades due to conflicts that would be pruned out of the + // final graph anyway. + // + // Note that even if we don't find a go version in mustSelect, it is possible + // that we will switch from unpruned to pruned (but not the other way around!) + // after applying the edits if we find a dependency that requires a high + // enough go version to trigger an upgrade. + rootPruning := orig.pruning + for _, m := range mustSelect { + if m.Path == "go" { + rootPruning = pruningForGoVersion(m.Version) + break + } else if m.Path == "toolchain" && pruningForGoVersion(gover.FromToolchain(m.Version)) == unpruned { + // We don't know exactly what go version we will end up at, but we know + // that it must be a version supported by the requested toolchain, and + // that toolchain does not support pruning. + // + // TODO(bcmills): 'go get' ought to reject explicit toolchain versions + // older than gover.GoStrictVersion. Once that is fixed, is this still + // needed? + rootPruning = unpruned + break + } + } + + if rootPruning != rs.pruning { + rs, err = convertPruning(ctx, rs, rootPruning) + if err != nil { + return orig, false, err + } + } + + // selectedRoot records the edited version (possibly "none") for each module + // path that would be a root in the edited requirements. + var selectedRoot map[string]string // module path → edited version + if rootPruning == pruned { + selectedRoot = maps.Clone(rs.maxRootVersion) + } else { + // In a module without graph pruning, modules that provide packages imported + // by the main module may either be explicit roots or implicit transitive + // dependencies. To the extent possible, we want to preserve those implicit + // dependencies, so we need to treat everything in the build list as + // potentially relevant — that is, as what would be a “root” in a module + // with graph pruning enabled. + mg, err := rs.Graph(ctx) + if err != nil { + // If we couldn't load the graph, we don't know what its requirements were + // to begin with, so we can't edit those requirements in a coherent way. + return orig, false, err + } + bl := mg.BuildList()[MainModules.Len():] + selectedRoot = make(map[string]string, len(bl)) + for _, m := range bl { + selectedRoot[m.Path] = m.Version + } + } + + for _, r := range tryUpgrade { + if v, ok := selectedRoot[r.Path]; ok && gover.ModCompare(r.Path, v, r.Version) >= 0 { + continue + } + if cfg.BuildV { + fmt.Fprintf(os.Stderr, "go: trying upgrade to %v\n", r) + } + selectedRoot[r.Path] = r.Version + } + + // conflicts is a list of conflicts that we cannot resolve without violating + // some version in mustSelect. It may be incomplete, but we want to report + // as many conflicts as we can so that the user can solve more of them at once. + var conflicts []Conflict + + // mustSelectVersion is an index of the versions in mustSelect. + mustSelectVersion := make(map[string]string, len(mustSelect)) + for _, r := range mustSelect { + if v, ok := mustSelectVersion[r.Path]; ok && v != r.Version { + prev := module.Version{Path: r.Path, Version: v} + if gover.ModCompare(r.Path, v, r.Version) > 0 { + conflicts = append(conflicts, Conflict{Path: []module.Version{prev}, Constraint: r}) + } else { + conflicts = append(conflicts, Conflict{Path: []module.Version{r}, Constraint: prev}) + } + continue + } + + mustSelectVersion[r.Path] = r.Version + selectedRoot[r.Path] = r.Version + } + + // We've indexed all of the data we need and we've computed the initial + // versions of the roots. Now we need to load the actual module graph and + // restore the invariant that every root is the selected version of its path. + // + // For 'go mod tidy' we would do that using expandGraph, which upgrades the + // roots until their requirements are internally consistent and then drops out + // the old roots. However, here we need to do more: we also need to make sure + // the modules in mustSelect don't get upgraded above their intended versions. + // To do that, we repeatedly walk the module graph, identify paths of + // requirements that result in versions that are too high, and downgrade the + // roots that lead to those paths. When no conflicts remain, we're done. + // + // Since we want to report accurate paths to each conflict, we don't drop out + // older-than-selected roots until the process completes. That might mean that + // we do some extra downgrades when they could be skipped, but for the benefit + // of being able to explain the reason for every downgrade that seems + // worthwhile. + // + // Graph pruning adds an extra wrinkle: a given node in the module graph + // may be reached from a root whose dependencies are pruned, and from a root + // whose dependencies are not pruned. It may be the case that the path from + // the unpruned root leads to a conflict, while the path from the pruned root + // prunes out the requirements that would lead to that conflict. + // So we need to track the two kinds of paths independently. + // They join back together at the roots of the graph: if a root r1 with pruned + // requirements depends on a root r2 with unpruned requirements, then + // selecting r1 would cause r2 to become a root and pull in all of its + // unpruned dependencies. + // + // The dqTracker type implements the logic for propagating conflict paths + // through the pruned and unpruned parts of the module graph. + // + // We make a best effort to fix incompatibilities, subject to two properties: + // + // 1. If the user runs 'go get' with a set of mutually-compatible module + // versions, we should accept those versions. + // + // 2. If we end up upgrading or downgrading a module, it should be + // clear why we did so. + // + // We don't try to find an optimal SAT solution, + // especially given the complex interactions with graph pruning. + + var ( + roots []module.Version // the current versions in selectedRoot, in sorted order + rootsDirty = true // true if roots does not match selectedRoot + ) + + // rejectedRoot records the set of module versions that have been disqualified + // as roots of the module graph. When downgrading due to a conflict or error, + // we skip any version that has already been rejected. + // + // NOTE(bcmills): I am not sure that the rejectedRoot map is really necessary, + // since we normally only downgrade roots or accept indirect upgrades to + // known-good versions. However, I am having trouble proving that accepting an + // indirect upgrade never introduces a conflict that leads to further + // downgrades. I really want to be able to prove that editRequirements + // terminates, and the easiest way to prove it is to add this map. + // + // Then the proof of termination is this: + // On every iteration where we mark the roots as dirty, we add some new module + // version to the map. The universe of module versions is finite, so we must + // eventually reach a state in which we do not add any version to the map. + // In that state, we either report a conflict or succeed in the edit. + rejectedRoot := map[module.Version]bool{} + + for rootsDirty && len(conflicts) == 0 { + roots = roots[:0] + for p, v := range selectedRoot { + if v != "none" { + roots = append(roots, module.Version{Path: p, Version: v}) + } + } + gover.ModSort(roots) + + // First, we extend the graph so that it includes the selected version + // of every root. The upgraded roots are in addition to the original + // roots, so we will have enough information to trace a path to each + // conflict we discover from one or more of the original roots. + mg, upgradedRoots, err := extendGraph(ctx, rootPruning, roots, selectedRoot) + if err != nil { + var tooNew *gover.TooNewError + if mg == nil || errors.As(err, &tooNew) { + return orig, false, err + } + // We're about to walk the entire extended module graph, so we will find + // any error then — and we will either try to resolve it by downgrading + // something or report it as a conflict with more detail. + } + + // extendedRootPruning is an index of the pruning used to load each root in + // the extended module graph. + extendedRootPruning := make(map[module.Version]modPruning, len(roots)+len(upgradedRoots)) + findPruning := func(m module.Version) modPruning { + if rootPruning == pruned { + summary, _ := mg.loadCache.Get(m) + if summary != nil && summary.pruning == unpruned { + return unpruned + } + } + return rootPruning + } + for _, m := range roots { + extendedRootPruning[m] = findPruning(m) + } + for m := range upgradedRoots { + extendedRootPruning[m] = findPruning(m) + } + + // Now check the resulting extended graph for errors and incompatibilities. + t := dqTracker{extendedRootPruning: extendedRootPruning} + mg.g.WalkBreadthFirst(func(m module.Version) { + if max, ok := mustSelectVersion[m.Path]; ok && gover.ModCompare(m.Path, m.Version, max) > 0 { + // m itself violates mustSelect, so it cannot appear in the module graph + // even if its transitive dependencies would be pruned out. + t.disqualify(m, pruned, dqState{dep: m}) + return + } + + summary, err := mg.loadCache.Get(m) + if err != nil && err != par.ErrCacheEntryNotFound { + // We can't determine the requirements of m, so we don't know whether + // they would be allowed. This may be a transient error reaching the + // repository, rather than a permanent error with the retrieved version. + // + // TODO(golang.org/issue/31730, golang.org/issue/30134): + // decide what to do based on the actual error. + t.disqualify(m, pruned, dqState{err: err}) + return + } + + reqs, ok := mg.RequiredBy(m) + if !ok { + // The dependencies of m do not appear in the module graph, so they + // can't be causing any problems this time. + return + } + + if summary == nil { + if m.Version != "" { + panic(fmt.Sprintf("internal error: %d reqs present for %v, but summary is nil", len(reqs), m)) + } + // m is the main module: we are editing its dependencies, so it cannot + // become disqualified. + return + } + + // Before we check for problems due to transitive dependencies, first + // check m's direct requirements. A requirement on a version r that + // violates mustSelect disqualifies m, even if the requirements of r are + // themselves pruned out. + for _, r := range reqs { + if max, ok := mustSelectVersion[r.Path]; ok && gover.ModCompare(r.Path, r.Version, max) > 0 { + t.disqualify(m, pruned, dqState{dep: r}) + return + } + } + for _, r := range reqs { + if !t.require(m, r) { + break + } + } + }) + + // We have now marked all of the versions in the graph that have conflicts, + // with a path to each conflict from one or more roots that introduce it. + // Now we need to identify those roots and change their versions + // (if possible) in order to resolve the conflicts. + rootsDirty = false + for _, m := range roots { + path, err := t.path(m, extendedRootPruning[m]) + if len(path) == 0 && err == nil { + continue // Nothing wrong with m; we can keep it. + } + + // path leads to a module with a problem: either it violates a constraint, + // or some error prevents us from determining whether it violates a + // constraint. We might end up logging or returning the conflict + // information, so go ahead and fill in the details about it. + conflict := Conflict{ + Path: path, + Err: err, + } + if err == nil { + var last module.Version = path[len(path)-1] + mustV, ok := mustSelectVersion[last.Path] + if !ok { + fmt.Fprintf(os.Stderr, "go: %v\n", conflict) + panic("internal error: found a version conflict, but no constraint it violates") + } + conflict.Constraint = module.Version{ + Path: last.Path, + Version: mustV, + } + } + + if v, ok := mustSelectVersion[m.Path]; ok && v == m.Version { + // m is in mustSelect, but is marked as disqualified due to a transitive + // dependency. + // + // In theory we could try removing module paths that don't appear in + // mustSelect (added by tryUpgrade or already present in rs) in order to + // get graph pruning to take effect, but (a) it is likely that 'go mod + // tidy' would re-add those roots and reintroduce unwanted upgrades, + // causing confusion, and (b) deciding which roots to try to eliminate + // would add a lot of complexity. + // + // Instead, we report the path to the conflict as an error. + // If users want to explicitly prune out nodes from the dependency + // graph, they can always add an explicit 'exclude' directive. + conflicts = append(conflicts, conflict) + continue + } + + // If m is not the selected version of its path, we have two options: we + // can either upgrade to the version that actually is selected (dropping m + // itself out of the bottom of the module graph), or we can try + // downgrading it. + // + // If the version we would be upgrading to is ok to use, we will just plan + // to do that and avoid the overhead of trying to find some lower version + // to downgrade to. + // + // However, it is possible that m depends on something that leads to its + // own upgrade, so if the upgrade isn't viable we should go ahead and try + // to downgrade (like with any other root). + if v := mg.Selected(m.Path); v != m.Version { + u := module.Version{Path: m.Path, Version: v} + uPruning, ok := t.extendedRootPruning[m] + if !ok { + fmt.Fprintf(os.Stderr, "go: %v\n", conflict) + panic(fmt.Sprintf("internal error: selected version of root %v is %v, but it was not expanded as a new root", m, u)) + } + if !t.check(u, uPruning).isDisqualified() && !rejectedRoot[u] { + // Applying the upgrade from m to u will resolve the conflict, + // so plan to do that if there are no other conflicts to resolve. + continue + } + } + + // Figure out what version of m's path was present before we started + // the edit. We want to make sure we consider keeping it as-is, + // even if it wouldn't normally be included. (For example, it might + // be a pseudo-version or pre-release.) + origMG, _ := orig.Graph(ctx) + origV := origMG.Selected(m.Path) + + if conflict.Err != nil && origV == m.Version { + // This version of m.Path was already in the module graph before we + // started editing, and the problem with it is that we can't load its + // (transitive) requirements. + // + // If this conflict was just one step in a longer chain of downgrades, + // then we would want to keep going past it until we find a version + // that doesn't have that problem. However, we only want to downgrade + // away from an *existing* requirement if we can confirm that it actually + // conflicts with mustSelect. (For example, we don't want + // 'go get -u ./...' to incidentally downgrade some dependency whose + // go.mod file is unavailable or has a bad checksum.) + conflicts = append(conflicts, conflict) + continue + } + + // We need to downgrade m's path to some lower version to try to resolve + // the conflict. Find the next-lowest candidate and apply it. + rejectedRoot[m] = true + prev := m + for { + prev, err = previousVersion(ctx, prev) + if gover.ModCompare(m.Path, m.Version, origV) > 0 && (gover.ModCompare(m.Path, prev.Version, origV) < 0 || err != nil) { + // previousVersion skipped over origV. Insert it into the order. + prev.Version = origV + } else if err != nil { + // We don't know the next downgrade to try. Give up. + return orig, false, err + } + if rejectedRoot[prev] { + // We already rejected prev in a previous round. + // To ensure that this algorithm terminates, don't try it again. + continue + } + pruning := rootPruning + if pruning == pruned { + if summary, err := mg.loadCache.Get(m); err == nil { + pruning = summary.pruning + } + } + if t.check(prev, pruning).isDisqualified() { + // We found a problem with prev this round that would also disqualify + // it as a root. Don't bother trying it next round. + rejectedRoot[prev] = true + continue + } + break + } + selectedRoot[m.Path] = prev.Version + rootsDirty = true + + // If this downgrade is potentially interesting, log the reason for it. + if conflict.Err != nil || cfg.BuildV { + var action string + if prev.Version == "none" { + action = fmt.Sprintf("removing %s", m) + } else if prev.Version == origV { + action = fmt.Sprintf("restoring %s", prev) + } else { + action = fmt.Sprintf("trying %s", prev) + } + fmt.Fprintf(os.Stderr, "go: %s\n\t%s\n", conflict.Summary(), action) + } + } + if rootsDirty { + continue + } + + // We didn't resolve any issues by downgrading, but we may still need to + // resolve some conflicts by locking in upgrades. Do that now. + // + // We don't do these upgrades until we're done downgrading because the + // downgrade process might reveal or remove conflicts (by changing which + // requirement edges are pruned out). + var upgradedFrom []module.Version // for logging only + for p, v := range selectedRoot { + if _, ok := mustSelectVersion[p]; !ok { + if actual := mg.Selected(p); actual != v { + if cfg.BuildV { + upgradedFrom = append(upgradedFrom, module.Version{Path: p, Version: v}) + } + selectedRoot[p] = actual + // Accepting the upgrade to m.Path might cause the selected versions + // of other modules to fall, because they were being increased by + // dependencies of m that are no longer present in the graph. + // + // TODO(bcmills): Can removing m as a root also cause the selected + // versions of other modules to rise? I think not: we're strictly + // removing non-root nodes from the module graph, which can't cause + // any root to decrease (because they're roots), and the dependencies + // of non-roots don't matter because they're either always unpruned or + // always pruned out. + // + // At any rate, it shouldn't cost much to reload the module graph one + // last time and confirm that it is stable. + rootsDirty = true + } + } + } + if rootsDirty { + if cfg.BuildV { + gover.ModSort(upgradedFrom) // Make logging deterministic. + for _, m := range upgradedFrom { + fmt.Fprintf(os.Stderr, "go: accepting indirect upgrade from %v to %s\n", m, selectedRoot[m.Path]) + } + } + continue + } + break + } + if len(conflicts) > 0 { + return orig, false, &ConstraintError{Conflicts: conflicts} + } + + if rootPruning == unpruned { + // An unpruned go.mod file lists only a subset of the requirements needed + // for building packages. Figure out which requirements need to be explicit. + var rootPaths []string + + // The modules in mustSelect are always promoted to be explicit. + for _, m := range mustSelect { + if m.Version != "none" && !MainModules.Contains(m.Path) { + rootPaths = append(rootPaths, m.Path) + } + } + + for _, m := range roots { + if v, ok := rs.rootSelected(m.Path); ok && (v == m.Version || rs.direct[m.Path]) { + // m.Path was formerly a root, and either its version hasn't changed or + // we believe that it provides a package directly imported by a package + // or test in the main module. For now we'll assume that it is still + // relevant enough to remain a root. If we actually load all of the + // packages and tests in the main module (which we are not doing here), + // we can revise the explicit roots at that point. + rootPaths = append(rootPaths, m.Path) + } + } + + roots, err = mvs.Req(MainModules.mustGetSingleMainModule(), rootPaths, &mvsReqs{roots: roots}) + if err != nil { + return nil, false, err + } + } + + changed = rootPruning != orig.pruning || !slices.Equal(roots, orig.rootModules) + if !changed { + // Because the roots we just computed are unchanged, the entire graph must + // be the same as it was before. Save the original rs, since we have + // probably already loaded its requirement graph. + return orig, false, nil + } + + // A module that is not even in the build list necessarily cannot provide + // any imported packages. Mark as direct only the direct modules that are + // still in the build list. (We assume that any module path that provided a + // direct import before the edit continues to do so after. There are a few + // edge cases where that can change, such as if a package moves into or out of + // a nested module or disappears entirely. If that happens, the user can run + // 'go mod tidy' to clean up the direct/indirect annotations.) + // + // TODO(bcmills): Would it make more sense to leave the direct map as-is + // but allow it to refer to modules that are no longer in the build list? + // That might complicate updateRoots, but it may be cleaner in other ways. + direct := make(map[string]bool, len(rs.direct)) + for _, m := range roots { + if rs.direct[m.Path] { + direct[m.Path] = true + } + } + edited = newRequirements(rootPruning, roots, direct) + + // If we ended up adding a dependency that upgrades our go version far enough + // to activate pruning, we must convert the edited Requirements in order to + // avoid dropping transitive dependencies from the build list the next time + // someone uses the updated go.mod file. + // + // Note that it isn't possible to go in the other direction (from pruned to + // unpruned) unless the "go" or "toolchain" module is explicitly listed in + // mustSelect, which we already handled at the very beginning of the edit. + // That is because the virtual "go" module only requires a "toolchain", + // and the "toolchain" module never requires anything else, which means that + // those two modules will never be downgraded due to a conflict with any other + // constraint. + if rootPruning == unpruned { + if v, ok := edited.rootSelected("go"); ok && pruningForGoVersion(v) == pruned { + // Since we computed the edit with the unpruned graph, and the pruned + // graph is a strict subset of the unpruned graph, this conversion + // preserves the exact (edited) build list that we already computed. + // + // However, it does that by shoving the whole build list into the roots of + // the graph. 'go get' will check for that sort of transition and log a + // message reminding the user how to clean up this mess we're about to + // make. 😅 + edited, err = convertPruning(ctx, edited, pruned) + if err != nil { + return orig, false, err + } + } + } + return edited, true, nil +} + +// extendGraph loads the module graph from roots, and iteratively extends it by +// unpruning the selected version of each module path that is a root in rs or in +// the roots slice until the graph reaches a fixed point. +// +// The graph is guaranteed to converge to a fixed point because unpruning a +// module version can only increase (never decrease) the selected versions, +// and the set of versions for each module is finite. +// +// The extended graph is useful for diagnosing version conflicts: for each +// selected module version, it can provide a complete path of requirements from +// some root to that version. +func extendGraph(ctx context.Context, rootPruning modPruning, roots []module.Version, selectedRoot map[string]string) (mg *ModuleGraph, upgradedRoot map[module.Version]bool, err error) { + for { + mg, err = readModGraph(ctx, rootPruning, roots, upgradedRoot) + // We keep on going even if err is non-nil until we reach a steady state. + // (Note that readModGraph returns a non-nil *ModuleGraph even in case of + // errors.) The caller may be able to fix the errors by adjusting versions, + // so we really want to return as complete a result as we can. + + if rootPruning == unpruned { + // Everything is already unpruned, so there isn't anything we can do to + // extend it further. + break + } + + nPrevRoots := len(upgradedRoot) + for p := range selectedRoot { + // Since p is a root path, when we fix up the module graph to be + // consistent with the selected versions, p will be promoted to a root, + // which will pull in its dependencies. Ensure that its dependencies are + // included in the module graph. + v := mg.g.Selected(p) + if v == "none" { + // Version “none” always has no requirements, so it doesn't need + // an explicit node in the module graph. + continue + } + m := module.Version{Path: p, Version: v} + if _, ok := mg.g.RequiredBy(m); !ok && !upgradedRoot[m] { + // The dependencies of the selected version of p were not loaded. + // Mark it as an upgrade so that we will load its dependencies + // in the next iteration. + // + // Note that we don't remove any of the existing roots, even if they are + // no longer the selected version: with graph pruning in effect this may + // leave some spurious dependencies in the graph, but it at least + // preserves enough of the graph to explain why each upgrade occurred: + // this way, we can report a complete path from the passed-in roots + // to every node in the module graph. + // + // This process is guaranteed to reach a fixed point: since we are only + // adding roots (never removing them), the selected version of each module + // can only increase, never decrease, and the set of module versions in the + // universe is finite. + if upgradedRoot == nil { + upgradedRoot = make(map[module.Version]bool) + } + upgradedRoot[m] = true + } + } + if len(upgradedRoot) == nPrevRoots { + break + } + } + + return mg, upgradedRoot, err +} + +type perPruning[T any] struct { + pruned T + unpruned T +} + +func (pp perPruning[T]) from(p modPruning) T { + if p == unpruned { + return pp.unpruned + } + return pp.pruned +} + +// A dqTracker tracks and propagates the reason that each module version +// cannot be included in the module graph. +type dqTracker struct { + // extendedRootPruning is the modPruning given the go.mod file for each root + // in the extended module graph. + extendedRootPruning map[module.Version]modPruning + + // dqReason records whether and why each each encountered version is + // disqualified in a pruned or unpruned context. + dqReason map[module.Version]perPruning[dqState] + + // requiring maps each not-yet-disqualified module version to the versions + // that would cause that module's requirements to be included in a pruned or + // unpruned context. If that version becomes disqualified, the + // disqualification will be propagated to all of the versions in the + // corresponding list. + // + // This map is similar to the module requirement graph, but includes more + // detail about whether a given dependency edge appears in a pruned or + // unpruned context. (Other commands do not need this level of detail.) + requiring map[module.Version][]module.Version +} + +// A dqState indicates whether and why a module version is “disqualified” from +// being used in a way that would incorporate its requirements. +// +// The zero dqState indicates that the module version is not known to be +// disqualified, either because it is ok or because we are currently traversing +// a cycle that includes it. +type dqState struct { + err error // if non-nil, disqualified because the requirements of the module could not be read + dep module.Version // disqualified because the module is or requires dep +} + +func (dq dqState) isDisqualified() bool { + return dq != dqState{} +} + +func (dq dqState) String() string { + if dq.err != nil { + return dq.err.Error() + } + if dq.dep != (module.Version{}) { + return dq.dep.String() + } + return "(no conflict)" +} + +// require records that m directly requires r, in case r becomes disqualified. +// (These edges are in the opposite direction from the edges in an mvs.Graph.) +// +// If r is already disqualified, require propagates the disqualification to m +// and returns the reason for the disqualification. +func (t *dqTracker) require(m, r module.Version) (ok bool) { + rdq := t.dqReason[r] + rootPruning, isRoot := t.extendedRootPruning[r] + if isRoot && rdq.from(rootPruning).isDisqualified() { + // When we pull in m's dependencies, we will have an edge from m to r, and r + // is disqualified (it is a root, which causes its problematic dependencies + // to always be included). So we cannot pull in m's dependencies at all: + // m is completely disqualified. + t.disqualify(m, pruned, dqState{dep: r}) + return false + } + + if dq := rdq.from(unpruned); dq.isDisqualified() { + t.disqualify(m, unpruned, dqState{dep: r}) + if _, ok := t.extendedRootPruning[m]; !ok { + // Since m is not a root, its dependencies can't be included in the pruned + // part of the module graph, and will never be disqualified from a pruned + // reason. We've already disqualified everything that matters. + return false + } + } + + // Record that m is a dependant of r, so that if r is later disqualified + // m will be disqualified as well. + if t.requiring == nil { + t.requiring = make(map[module.Version][]module.Version) + } + t.requiring[r] = append(t.requiring[r], m) + return true +} + +// disqualify records why the dependencies of m cannot be included in the module +// graph if reached from a part of the graph with the given pruning. +// +// Since the pruned graph is a subgraph of the unpruned graph, disqualifying a +// module from a pruned part of the graph also disqualifies it in the unpruned +// parts. +func (t *dqTracker) disqualify(m module.Version, fromPruning modPruning, reason dqState) { + if !reason.isDisqualified() { + panic("internal error: disqualify called with a non-disqualifying dqState") + } + + dq := t.dqReason[m] + if dq.from(fromPruning).isDisqualified() { + return // Already disqualified for some other reason; don't overwrite it. + } + rootPruning, isRoot := t.extendedRootPruning[m] + if fromPruning == pruned { + dq.pruned = reason + if !dq.unpruned.isDisqualified() { + // Since the pruned graph of m is a subgraph of the unpruned graph, if it + // is disqualified due to something in the pruned graph, it is certainly + // disqualified in the unpruned graph from the same reason. + dq.unpruned = reason + } + } else { + dq.unpruned = reason + if dq.pruned.isDisqualified() { + panic(fmt.Sprintf("internal error: %v is marked as disqualified when pruned, but not when unpruned", m)) + } + if isRoot && rootPruning == unpruned { + // Since m is a root that is always unpruned, any other roots — even + // pruned ones! — that cause it to be selected would also cause the reason + // for is disqualification to be included in the module graph. + dq.pruned = reason + } + } + if t.dqReason == nil { + t.dqReason = make(map[module.Version]perPruning[dqState]) + } + t.dqReason[m] = dq + + if isRoot && (fromPruning == pruned || rootPruning == unpruned) { + // Either m is disqualified even when its dependencies are pruned, + // or m's go.mod file causes its dependencies to *always* be unpruned. + // Everything that depends on it must be disqualified. + for _, p := range t.requiring[m] { + t.disqualify(p, pruned, dqState{dep: m}) + // Note that since the pruned graph is a subset of the unpruned graph, + // disqualifying p in the pruned graph also disqualifies it in the + // unpruned graph. + } + // Everything in t.requiring[m] is now fully disqualified. + // We won't need to use it again. + delete(t.requiring, m) + return + } + + // Either m is not a root, or it is a pruned root but only being disqualified + // when reached from the unpruned parts of the module graph. + // Either way, the reason for this disqualification is only visible to the + // unpruned parts of the module graph. + for _, p := range t.requiring[m] { + t.disqualify(p, unpruned, dqState{dep: m}) + } + if !isRoot { + // Since m is not a root, its dependencies can't be included in the pruned + // part of the module graph, and will never be disqualified from a pruned + // reason. We've already disqualified everything that matters. + delete(t.requiring, m) + } +} + +// check reports whether m is disqualified in the given pruning context. +func (t *dqTracker) check(m module.Version, pruning modPruning) dqState { + return t.dqReason[m].from(pruning) +} + +// path returns the path from m to the reason it is disqualified, which may be +// either a module that violates constraints or an error in loading +// requirements. +// +// If m is not disqualified, path returns (nil, nil). +func (t *dqTracker) path(m module.Version, pruning modPruning) (path []module.Version, err error) { + for { + dq := t.dqReason[m].from(pruning) + if !dq.isDisqualified() { + return path, nil + } + path = append(path, m) + if dq.err != nil || dq.dep == m { + return path, dq.err // m itself is the conflict. + } + m = dq.dep + } +} diff --git a/src/cmd/go/internal/modload/help.go b/src/cmd/go/internal/modload/help.go new file mode 100644 index 0000000..886ad62 --- /dev/null +++ b/src/cmd/go/internal/modload/help.go @@ -0,0 +1,64 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import "cmd/go/internal/base" + +var HelpModules = &base.Command{ + UsageLine: "modules", + Short: "modules, module versions, and more", + Long: ` +Modules are how Go manages dependencies. + +A module is a collection of packages that are released, versioned, and +distributed together. Modules may be downloaded directly from version control +repositories or from module proxy servers. + +For a series of tutorials on modules, see +https://golang.org/doc/tutorial/create-module. + +For a detailed reference on modules, see https://golang.org/ref/mod. + +By default, the go command may download modules from https://proxy.golang.org. +It may authenticate modules using the checksum database at +https://sum.golang.org. Both services are operated by the Go team at Google. +The privacy policies for these services are available at +https://proxy.golang.org/privacy and https://sum.golang.org/privacy, +respectively. + +The go command's download behavior may be configured using GOPROXY, GOSUMDB, +GOPRIVATE, and other environment variables. See 'go help environment' +and https://golang.org/ref/mod#private-module-privacy for more information. + `, +} + +var HelpGoMod = &base.Command{ + UsageLine: "go.mod", + Short: "the go.mod file", + Long: ` +A module version is defined by a tree of source files, with a go.mod +file in its root. When the go command is run, it looks in the current +directory and then successive parent directories to find the go.mod +marking the root of the main (current) module. + +The go.mod file format is described in detail at +https://golang.org/ref/mod#go-mod-file. + +To create a new go.mod file, use 'go mod init'. For details see +'go help mod init' or https://golang.org/ref/mod#go-mod-init. + +To add missing module requirements or remove unneeded requirements, +use 'go mod tidy'. For details, see 'go help mod tidy' or +https://golang.org/ref/mod#go-mod-tidy. + +To add, upgrade, downgrade, or remove a specific module requirement, use +'go get'. For details, see 'go help module-get' or +https://golang.org/ref/mod#go-get. + +To make other changes or to parse go.mod as JSON for use by other tools, +use 'go mod edit'. See 'go help mod edit' or +https://golang.org/ref/mod#go-mod-edit. + `, +} diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go new file mode 100644 index 0000000..7cd5fcf --- /dev/null +++ b/src/cmd/go/internal/modload/import.go @@ -0,0 +1,784 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "context" + "errors" + "fmt" + "go/build" + "io/fs" + "os" + pathpkg "path" + "path/filepath" + "sort" + "strings" + + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/gover" + "cmd/go/internal/modfetch" + "cmd/go/internal/modindex" + "cmd/go/internal/par" + "cmd/go/internal/search" + "cmd/go/internal/str" + + "golang.org/x/mod/module" +) + +type ImportMissingError struct { + Path string + Module module.Version + QueryErr error + + ImportingMainModule module.Version + + // isStd indicates whether we would expect to find the package in the standard + // library. This is normally true for all dotless import paths, but replace + // directives can cause us to treat the replaced paths as also being in + // modules. + isStd bool + + // importerGoVersion is the version the module containing the import error + // specified. It is only set when isStd is true. + importerGoVersion string + + // replaced the highest replaced version of the module where the replacement + // contains the package. replaced is only set if the replacement is unused. + replaced module.Version + + // newMissingVersion is set to a newer version of Module if one is present + // in the build list. When set, we can't automatically upgrade. + newMissingVersion string +} + +func (e *ImportMissingError) Error() string { + if e.Module.Path == "" { + if e.isStd { + msg := fmt.Sprintf("package %s is not in std (%s)", e.Path, filepath.Join(cfg.GOROOT, "src", e.Path)) + if e.importerGoVersion != "" { + msg += fmt.Sprintf("\nnote: imported by a module that requires go %s", e.importerGoVersion) + } + return msg + } + if e.QueryErr != nil && e.QueryErr != ErrNoModRoot { + return fmt.Sprintf("cannot find module providing package %s: %v", e.Path, e.QueryErr) + } + if cfg.BuildMod == "mod" || (cfg.BuildMod == "readonly" && allowMissingModuleImports) { + return "cannot find module providing package " + e.Path + } + + if e.replaced.Path != "" { + suggestArg := e.replaced.Path + if !module.IsZeroPseudoVersion(e.replaced.Version) { + suggestArg = e.replaced.String() + } + return fmt.Sprintf("module %s provides package %s and is replaced but not required; to add it:\n\tgo get %s", e.replaced.Path, e.Path, suggestArg) + } + + message := fmt.Sprintf("no required module provides package %s", e.Path) + if e.QueryErr != nil { + return fmt.Sprintf("%s: %v", message, e.QueryErr) + } + if e.ImportingMainModule.Path != "" && e.ImportingMainModule != MainModules.ModContainingCWD() { + return fmt.Sprintf("%s; to add it:\n\tcd %s\n\tgo get %s", message, MainModules.ModRoot(e.ImportingMainModule), e.Path) + } + return fmt.Sprintf("%s; to add it:\n\tgo get %s", message, e.Path) + } + + if e.newMissingVersion != "" { + return fmt.Sprintf("package %s provided by %s at latest version %s but not at required version %s", e.Path, e.Module.Path, e.Module.Version, e.newMissingVersion) + } + + return fmt.Sprintf("missing module for import: %s@%s provides %s", e.Module.Path, e.Module.Version, e.Path) +} + +func (e *ImportMissingError) Unwrap() error { + return e.QueryErr +} + +func (e *ImportMissingError) ImportPath() string { + return e.Path +} + +// An AmbiguousImportError indicates an import of a package found in multiple +// modules in the build list, or found in both the main module and its vendor +// directory. +type AmbiguousImportError struct { + importPath string + Dirs []string + Modules []module.Version // Either empty or 1:1 with Dirs. +} + +func (e *AmbiguousImportError) ImportPath() string { + return e.importPath +} + +func (e *AmbiguousImportError) Error() string { + locType := "modules" + if len(e.Modules) == 0 { + locType = "directories" + } + + var buf strings.Builder + fmt.Fprintf(&buf, "ambiguous import: found package %s in multiple %s:", e.importPath, locType) + + for i, dir := range e.Dirs { + buf.WriteString("\n\t") + if i < len(e.Modules) { + m := e.Modules[i] + buf.WriteString(m.Path) + if m.Version != "" { + fmt.Fprintf(&buf, " %s", m.Version) + } + fmt.Fprintf(&buf, " (%s)", dir) + } else { + buf.WriteString(dir) + } + } + + return buf.String() +} + +// A DirectImportFromImplicitDependencyError indicates a package directly +// imported by a package or test in the main module that is satisfied by a +// dependency that is not explicit in the main module's go.mod file. +type DirectImportFromImplicitDependencyError struct { + ImporterPath string + ImportedPath string + Module module.Version +} + +func (e *DirectImportFromImplicitDependencyError) Error() string { + return fmt.Sprintf("package %s imports %s from implicitly required module; to add missing requirements, run:\n\tgo get %s@%s", e.ImporterPath, e.ImportedPath, e.Module.Path, e.Module.Version) +} + +func (e *DirectImportFromImplicitDependencyError) ImportPath() string { + return e.ImporterPath +} + +// ImportMissingSumError is reported in readonly mode when we need to check +// if a module contains a package, but we don't have a sum for its .zip file. +// We might need sums for multiple modules to verify the package is unique. +// +// TODO(#43653): consolidate multiple errors of this type into a single error +// that suggests a 'go get' command for root packages that transitively import +// packages from modules with missing sums. load.CheckPackageErrors would be +// a good place to consolidate errors, but we'll need to attach the import +// stack here. +type ImportMissingSumError struct { + importPath string + found bool + mods []module.Version + importer, importerVersion string // optional, but used for additional context + importerIsTest bool +} + +func (e *ImportMissingSumError) Error() string { + var importParen string + if e.importer != "" { + importParen = fmt.Sprintf(" (imported by %s)", e.importer) + } + var message string + if e.found { + message = fmt.Sprintf("missing go.sum entry needed to verify package %s%s is provided by exactly one module", e.importPath, importParen) + } else { + message = fmt.Sprintf("missing go.sum entry for module providing package %s%s", e.importPath, importParen) + } + var hint string + if e.importer == "" { + // Importing package is unknown, or the missing package was named on the + // command line. Recommend 'go mod download' for the modules that could + // provide the package, since that shouldn't change go.mod. + if len(e.mods) > 0 { + args := make([]string, len(e.mods)) + for i, mod := range e.mods { + args[i] = mod.Path + } + hint = fmt.Sprintf("; to add:\n\tgo mod download %s", strings.Join(args, " ")) + } + } else { + // Importing package is known (common case). Recommend 'go get' on the + // current version of the importing package. + tFlag := "" + if e.importerIsTest { + tFlag = " -t" + } + version := "" + if e.importerVersion != "" { + version = "@" + e.importerVersion + } + hint = fmt.Sprintf("; to add:\n\tgo get%s %s%s", tFlag, e.importer, version) + } + return message + hint +} + +func (e *ImportMissingSumError) ImportPath() string { + return e.importPath +} + +type invalidImportError struct { + importPath string + err error +} + +func (e *invalidImportError) ImportPath() string { + return e.importPath +} + +func (e *invalidImportError) Error() string { + return e.err.Error() +} + +func (e *invalidImportError) Unwrap() error { + return e.err +} + +// importFromModules finds the module and directory in the dependency graph of +// rs containing the package with the given import path. If mg is nil, +// importFromModules attempts to locate the module using only the main module +// and the roots of rs before it loads the full graph. +// +// The answer must be unique: importFromModules returns an error if multiple +// modules are observed to provide the same package. +// +// importFromModules can return a module with an empty m.Path, for packages in +// the standard library. +// +// importFromModules can return an empty directory string, for fake packages +// like "C" and "unsafe". +// +// If the package is not present in any module selected from the requirement +// graph, importFromModules returns an *ImportMissingError. +// +// If the package is present in exactly one module, importFromModules will +// return the module, its root directory, and a list of other modules that +// lexically could have provided the package but did not. +// +// If skipModFile is true, the go.mod file for the package is not loaded. This +// allows 'go mod tidy' to preserve a minor checksum-preservation bug +// (https://go.dev/issue/56222) for modules with 'go' versions between 1.17 and +// 1.20, preventing unnecessary go.sum churn and network access in those +// modules. +func importFromModules(ctx context.Context, path string, rs *Requirements, mg *ModuleGraph, skipModFile bool) (m module.Version, modroot, dir string, altMods []module.Version, err error) { + invalidf := func(format string, args ...interface{}) (module.Version, string, string, []module.Version, error) { + return module.Version{}, "", "", nil, &invalidImportError{ + importPath: path, + err: fmt.Errorf(format, args...), + } + } + + if strings.Contains(path, "@") { + return invalidf("import path %q should not have @version", path) + } + if build.IsLocalImport(path) { + return invalidf("%q is relative, but relative import paths are not supported in module mode", path) + } + if filepath.IsAbs(path) { + return invalidf("%q is not a package path; see 'go help packages'", path) + } + if search.IsMetaPackage(path) { + return invalidf("%q is not an importable package; see 'go help packages'", path) + } + + if path == "C" { + // There's no directory for import "C". + return module.Version{}, "", "", nil, nil + } + // Before any further lookup, check that the path is valid. + if err := module.CheckImportPath(path); err != nil { + return module.Version{}, "", "", nil, &invalidImportError{importPath: path, err: err} + } + + // Check each module on the build list. + var dirs, roots []string + var mods []module.Version + + // Is the package in the standard library? + pathIsStd := search.IsStandardImportPath(path) + if pathIsStd && modindex.IsStandardPackage(cfg.GOROOT, cfg.BuildContext.Compiler, path) { + for _, mainModule := range MainModules.Versions() { + if MainModules.InGorootSrc(mainModule) { + if dir, ok, err := dirInModule(path, MainModules.PathPrefix(mainModule), MainModules.ModRoot(mainModule), true); err != nil { + return module.Version{}, MainModules.ModRoot(mainModule), dir, nil, err + } else if ok { + return mainModule, MainModules.ModRoot(mainModule), dir, nil, nil + } + } + } + dir := filepath.Join(cfg.GOROOTsrc, path) + modroot = cfg.GOROOTsrc + if str.HasPathPrefix(path, "cmd") { + modroot = filepath.Join(cfg.GOROOTsrc, "cmd") + } + dirs = append(dirs, dir) + roots = append(roots, modroot) + mods = append(mods, module.Version{}) + } + // -mod=vendor is special. + // Everything must be in the main modules or the main module's or workspace's vendor directory. + if cfg.BuildMod == "vendor" { + var mainErr error + for _, mainModule := range MainModules.Versions() { + modRoot := MainModules.ModRoot(mainModule) + if modRoot != "" { + dir, mainOK, err := dirInModule(path, MainModules.PathPrefix(mainModule), modRoot, true) + if mainErr == nil { + mainErr = err + } + if mainOK { + mods = append(mods, mainModule) + dirs = append(dirs, dir) + roots = append(roots, modRoot) + } + } + } + + if HasModRoot() { + vendorDir := VendorDir() + dir, vendorOK, _ := dirInModule(path, "", vendorDir, false) + if vendorOK { + readVendorList(vendorDir) + // TODO(#60922): It's possible for a package to manually have been added to the + // vendor directory, causing the dirInModule to succeed, but no vendorPkgModule + // to exist, causing an empty module path to be reported. Do better checking + // here. + mods = append(mods, vendorPkgModule[path]) + dirs = append(dirs, dir) + roots = append(roots, vendorDir) + } + } + + if len(dirs) > 1 { + return module.Version{}, "", "", nil, &AmbiguousImportError{importPath: path, Dirs: dirs} + } + + if mainErr != nil { + return module.Version{}, "", "", nil, mainErr + } + + if len(dirs) == 0 { + return module.Version{}, "", "", nil, &ImportMissingError{Path: path} + } + + return mods[0], roots[0], dirs[0], nil, nil + } + + // Iterate over possible modules for the path, not all selected modules. + // Iterating over selected modules would make the overall loading time + // O(M × P) for M modules providing P imported packages, whereas iterating + // over path prefixes is only O(P × k) with maximum path depth k. For + // large projects both M and P may be very large (note that M ≤ P), but k + // will tend to remain smallish (if for no other reason than filesystem + // path limitations). + // + // We perform this iteration either one or two times. If mg is initially nil, + // then we first attempt to load the package using only the main module and + // its root requirements. If that does not identify the package, or if mg is + // already non-nil, then we attempt to load the package using the full + // requirements in mg. + for { + var sumErrMods, altMods []module.Version + for prefix := path; prefix != "."; prefix = pathpkg.Dir(prefix) { + if gover.IsToolchain(prefix) { + // Do not use the synthetic "go" module for "go/ast". + continue + } + var ( + v string + ok bool + ) + if mg == nil { + v, ok = rs.rootSelected(prefix) + } else { + v, ok = mg.Selected(prefix), true + } + if !ok || v == "none" { + continue + } + m := module.Version{Path: prefix, Version: v} + + root, isLocal, err := fetch(ctx, m) + if err != nil { + if sumErr := (*sumMissingError)(nil); errors.As(err, &sumErr) { + // We are missing a sum needed to fetch a module in the build list. + // We can't verify that the package is unique, and we may not find + // the package at all. Keep checking other modules to decide which + // error to report. Multiple sums may be missing if we need to look in + // multiple nested modules to resolve the import; we'll report them all. + sumErrMods = append(sumErrMods, m) + continue + } + // Report fetch error. + // Note that we don't know for sure this module is necessary, + // but it certainly _could_ provide the package, and even if we + // continue the loop and find the package in some other module, + // we need to look at this module to make sure the import is + // not ambiguous. + return module.Version{}, "", "", nil, err + } + if dir, ok, err := dirInModule(path, m.Path, root, isLocal); err != nil { + return module.Version{}, "", "", nil, err + } else if ok { + mods = append(mods, m) + roots = append(roots, root) + dirs = append(dirs, dir) + } else { + altMods = append(altMods, m) + } + } + + if len(mods) > 1 { + // We produce the list of directories from longest to shortest candidate + // module path, but the AmbiguousImportError should report them from + // shortest to longest. Reverse them now. + for i := 0; i < len(mods)/2; i++ { + j := len(mods) - 1 - i + mods[i], mods[j] = mods[j], mods[i] + roots[i], roots[j] = roots[j], roots[i] + dirs[i], dirs[j] = dirs[j], dirs[i] + } + return module.Version{}, "", "", nil, &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods} + } + + if len(sumErrMods) > 0 { + for i := 0; i < len(sumErrMods)/2; i++ { + j := len(sumErrMods) - 1 - i + sumErrMods[i], sumErrMods[j] = sumErrMods[j], sumErrMods[i] + } + return module.Version{}, "", "", nil, &ImportMissingSumError{ + importPath: path, + mods: sumErrMods, + found: len(mods) > 0, + } + } + + if len(mods) == 1 { + // We've found the unique module containing the package. + // However, in order to actually compile it we need to know what + // Go language version to use, which requires its go.mod file. + // + // If the module graph is pruned and this is a test-only dependency + // of a package in "all", we didn't necessarily load that file + // when we read the module graph, so do it now to be sure. + if !skipModFile && cfg.BuildMod != "vendor" && mods[0].Path != "" && !MainModules.Contains(mods[0].Path) { + if _, err := goModSummary(mods[0]); err != nil { + return module.Version{}, "", "", nil, err + } + } + return mods[0], roots[0], dirs[0], altMods, nil + } + + if mg != nil { + // We checked the full module graph and still didn't find the + // requested package. + var queryErr error + if !HasModRoot() { + queryErr = ErrNoModRoot + } + return module.Version{}, "", "", nil, &ImportMissingError{Path: path, QueryErr: queryErr, isStd: pathIsStd} + } + + // So far we've checked the root dependencies. + // Load the full module graph and try again. + mg, err = rs.Graph(ctx) + if err != nil { + // We might be missing one or more transitive (implicit) dependencies from + // the module graph, so we can't return an ImportMissingError here — one + // of the missing modules might actually contain the package in question, + // in which case we shouldn't go looking for it in some new dependency. + return module.Version{}, "", "", nil, err + } + } +} + +// queryImport attempts to locate a module that can be added to the current +// build list to provide the package with the given import path. +// +// Unlike QueryPattern, queryImport prefers to add a replaced version of a +// module *before* checking the proxies for a version to add. +func queryImport(ctx context.Context, path string, rs *Requirements) (module.Version, error) { + // To avoid spurious remote fetches, try the latest replacement for each + // module (golang.org/issue/26241). + var mods []module.Version + if MainModules != nil { // TODO(#48912): Ensure MainModules exists at this point, and remove the check. + for mp, mv := range MainModules.HighestReplaced() { + if !maybeInModule(path, mp) { + continue + } + if mv == "" { + // The only replacement is a wildcard that doesn't specify a version, so + // synthesize a pseudo-version with an appropriate major version and a + // timestamp below any real timestamp. That way, if the main module is + // used from within some other module, the user will be able to upgrade + // the requirement to any real version they choose. + if _, pathMajor, ok := module.SplitPathVersion(mp); ok && len(pathMajor) > 0 { + mv = module.ZeroPseudoVersion(pathMajor[1:]) + } else { + mv = module.ZeroPseudoVersion("v0") + } + } + mg, err := rs.Graph(ctx) + if err != nil { + return module.Version{}, err + } + if gover.ModCompare(mp, mg.Selected(mp), mv) >= 0 { + // We can't resolve the import by adding mp@mv to the module graph, + // because the selected version of mp is already at least mv. + continue + } + mods = append(mods, module.Version{Path: mp, Version: mv}) + } + } + + // Every module path in mods is a prefix of the import path. + // As in QueryPattern, prefer the longest prefix that satisfies the import. + sort.Slice(mods, func(i, j int) bool { + return len(mods[i].Path) > len(mods[j].Path) + }) + for _, m := range mods { + root, isLocal, err := fetch(ctx, m) + if err != nil { + if sumErr := (*sumMissingError)(nil); errors.As(err, &sumErr) { + return module.Version{}, &ImportMissingSumError{importPath: path} + } + return module.Version{}, err + } + if _, ok, err := dirInModule(path, m.Path, root, isLocal); err != nil { + return m, err + } else if ok { + if cfg.BuildMod == "readonly" { + return module.Version{}, &ImportMissingError{Path: path, replaced: m} + } + return m, nil + } + } + if len(mods) > 0 && module.CheckPath(path) != nil { + // The package path is not valid to fetch remotely, + // so it can only exist in a replaced module, + // and we know from the above loop that it is not. + replacement := Replacement(mods[0]) + return module.Version{}, &PackageNotInModuleError{ + Mod: mods[0], + Query: "latest", + Pattern: path, + Replacement: replacement, + } + } + + if search.IsStandardImportPath(path) { + // This package isn't in the standard library, isn't in any module already + // in the build list, and isn't in any other module that the user has + // shimmed in via a "replace" directive. + // Moreover, the import path is reserved for the standard library, so + // QueryPattern cannot possibly find a module containing this package. + // + // Instead of trying QueryPattern, report an ImportMissingError immediately. + return module.Version{}, &ImportMissingError{Path: path, isStd: true} + } + + if (cfg.BuildMod == "readonly" || cfg.BuildMod == "vendor") && !allowMissingModuleImports { + // In readonly mode, we can't write go.mod, so we shouldn't try to look up + // the module. If readonly mode was enabled explicitly, include that in + // the error message. + // In vendor mode, we cannot use the network or module cache, so we + // shouldn't try to look up the module + var queryErr error + if cfg.BuildModExplicit { + queryErr = fmt.Errorf("import lookup disabled by -mod=%s", cfg.BuildMod) + } else if cfg.BuildModReason != "" { + queryErr = fmt.Errorf("import lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) + } + return module.Version{}, &ImportMissingError{Path: path, QueryErr: queryErr} + } + + // Look up module containing the package, for addition to the build list. + // Goal is to determine the module, download it to dir, + // and return m, dir, ImportMissingError. + fmt.Fprintf(os.Stderr, "go: finding module for package %s\n", path) + + mg, err := rs.Graph(ctx) + if err != nil { + return module.Version{}, err + } + + candidates, err := QueryPackages(ctx, path, "latest", mg.Selected, CheckAllowed) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // Return "cannot find module providing package […]" instead of whatever + // low-level error QueryPattern produced. + return module.Version{}, &ImportMissingError{Path: path, QueryErr: err} + } else { + return module.Version{}, err + } + } + + candidate0MissingVersion := "" + for i, c := range candidates { + if v := mg.Selected(c.Mod.Path); gover.ModCompare(c.Mod.Path, v, c.Mod.Version) > 0 { + // QueryPattern proposed that we add module c.Mod to provide the package, + // but we already depend on a newer version of that module (and that + // version doesn't have the package). + // + // This typically happens when a package is present at the "@latest" + // version (e.g., v1.0.0) of a module, but we have a newer version + // of the same module in the build list (e.g., v1.0.1-beta), and + // the package is not present there. + if i == 0 { + candidate0MissingVersion = v + } + continue + } + return c.Mod, nil + } + return module.Version{}, &ImportMissingError{ + Path: path, + Module: candidates[0].Mod, + newMissingVersion: candidate0MissingVersion, + } +} + +// maybeInModule reports whether, syntactically, +// a package with the given import path could be supplied +// by a module with the given module path (mpath). +func maybeInModule(path, mpath string) bool { + return mpath == path || + len(path) > len(mpath) && path[len(mpath)] == '/' && path[:len(mpath)] == mpath +} + +var ( + haveGoModCache par.Cache[string, bool] // dir → bool + haveGoFilesCache par.ErrCache[string, bool] // dir → haveGoFiles +) + +// dirInModule locates the directory that would hold the package named by the given path, +// if it were in the module with module path mpath and root mdir. +// If path is syntactically not within mpath, +// or if mdir is a local file tree (isLocal == true) and the directory +// that would hold path is in a sub-module (covered by a go.mod below mdir), +// dirInModule returns "", false, nil. +// +// Otherwise, dirInModule returns the name of the directory where +// Go source files would be expected, along with a boolean indicating +// whether there are in fact Go source files in that directory. +// A non-nil error indicates that the existence of the directory and/or +// source files could not be determined, for example due to a permission error. +func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFiles bool, err error) { + // Determine where to expect the package. + if path == mpath { + dir = mdir + } else if mpath == "" { // vendor directory + dir = filepath.Join(mdir, path) + } else if len(path) > len(mpath) && path[len(mpath)] == '/' && path[:len(mpath)] == mpath { + dir = filepath.Join(mdir, path[len(mpath)+1:]) + } else { + return "", false, nil + } + + // Check that there aren't other modules in the way. + // This check is unnecessary inside the module cache + // and important to skip in the vendor directory, + // where all the module trees have been overlaid. + // So we only check local module trees + // (the main module, and any directory trees pointed at by replace directives). + if isLocal { + for d := dir; d != mdir && len(d) > len(mdir); { + haveGoMod := haveGoModCache.Do(d, func() bool { + fi, err := fsys.Stat(filepath.Join(d, "go.mod")) + return err == nil && !fi.IsDir() + }) + + if haveGoMod { + return "", false, nil + } + parent := filepath.Dir(d) + if parent == d { + // Break the loop, as otherwise we'd loop + // forever if d=="." and mdir=="". + break + } + d = parent + } + } + + // Now committed to returning dir (not ""). + + // Are there Go source files in the directory? + // We don't care about build tags, not even "go:build ignore". + // We're just looking for a plausible directory. + haveGoFiles, err = haveGoFilesCache.Do(dir, func() (bool, error) { + // modindex.GetPackage will return ErrNotIndexed for any directories which + // are reached through a symlink, so that they will be handled by + // fsys.IsDirWithGoFiles below. + if ip, err := modindex.GetPackage(mdir, dir); err == nil { + return ip.IsDirWithGoFiles() + } else if !errors.Is(err, modindex.ErrNotIndexed) { + return false, err + } + return fsys.IsDirWithGoFiles(dir) + }) + + return dir, haveGoFiles, err +} + +// fetch downloads the given module (or its replacement) +// and returns its location. +// +// The isLocal return value reports whether the replacement, +// if any, is local to the filesystem. +func fetch(ctx context.Context, mod module.Version) (dir string, isLocal bool, err error) { + if modRoot := MainModules.ModRoot(mod); modRoot != "" { + return modRoot, true, nil + } + if r := Replacement(mod); r.Path != "" { + if r.Version == "" { + dir = r.Path + if !filepath.IsAbs(dir) { + dir = filepath.Join(replaceRelativeTo(), dir) + } + // Ensure that the replacement directory actually exists: + // dirInModule does not report errors for missing modules, + // so if we don't report the error now, later failures will be + // very mysterious. + if _, err := fsys.Stat(dir); err != nil { + // TODO(bcmills): We should also read dir/go.mod here and check its Go version, + // and return a gover.TooNewError if appropriate. + + if os.IsNotExist(err) { + // Semantically the module version itself “exists” — we just don't + // have its source code. Remove the equivalence to os.ErrNotExist, + // and make the message more concise while we're at it. + err = fmt.Errorf("replacement directory %s does not exist", r.Path) + } else { + err = fmt.Errorf("replacement directory %s: %w", r.Path, err) + } + return dir, true, module.VersionError(mod, err) + } + return dir, true, nil + } + mod = r + } + + if mustHaveSums() && !modfetch.HaveSum(mod) { + return "", false, module.VersionError(mod, &sumMissingError{}) + } + + dir, err = modfetch.Download(ctx, mod) + return dir, false, err +} + +// mustHaveSums reports whether we require that all checksums +// needed to load or build packages are already present in the go.sum file. +func mustHaveSums() bool { + return HasModRoot() && cfg.BuildMod == "readonly" && !inWorkspaceMode() +} + +type sumMissingError struct { + suggestion string +} + +func (e *sumMissingError) Error() string { + return "missing go.sum entry" + e.suggestion +} diff --git a/src/cmd/go/internal/modload/import_test.go b/src/cmd/go/internal/modload/import_test.go new file mode 100644 index 0000000..eb4f5d6 --- /dev/null +++ b/src/cmd/go/internal/modload/import_test.go @@ -0,0 +1,97 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "context" + "internal/testenv" + "regexp" + "strings" + "testing" + + "golang.org/x/mod/module" +) + +var importTests = []struct { + path string + m module.Version + err string +}{ + { + path: "golang.org/x/net/context", + m: module.Version{ + Path: "golang.org/x/net", + }, + }, + { + path: "golang.org/x/net", + err: `module golang.org/x/net@.* found \(v[01]\.\d+\.\d+\), but does not contain package golang.org/x/net`, + }, + { + path: "golang.org/x/text", + m: module.Version{ + Path: "golang.org/x/text", + }, + }, + { + path: "github.com/rsc/quote/buggy", + m: module.Version{ + Path: "github.com/rsc/quote", + Version: "v1.5.2", + }, + }, + { + path: "github.com/rsc/quote", + m: module.Version{ + Path: "github.com/rsc/quote", + Version: "v1.5.2", + }, + }, + { + path: "golang.org/x/foo/bar", + err: "cannot find module providing package golang.org/x/foo/bar", + }, +} + +func TestQueryImport(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + testenv.MustHaveExecPath(t, "git") + + oldAllowMissingModuleImports := allowMissingModuleImports + oldRootMode := RootMode + defer func() { + allowMissingModuleImports = oldAllowMissingModuleImports + RootMode = oldRootMode + }() + allowMissingModuleImports = true + RootMode = NoRoot + + ctx := context.Background() + rs := LoadModFile(ctx) + + for _, tt := range importTests { + t.Run(strings.ReplaceAll(tt.path, "/", "_"), func(t *testing.T) { + // Note that there is no build list, so Import should always fail. + m, err := queryImport(ctx, tt.path, rs) + + if tt.err == "" { + if err != nil { + t.Fatalf("queryImport(_, %q): %v", tt.path, err) + } + } else { + if err == nil { + t.Fatalf("queryImport(_, %q) = %v, nil; expected error", tt.path, m) + } + if !regexp.MustCompile(tt.err).MatchString(err.Error()) { + t.Fatalf("queryImport(_, %q): error %q, want error matching %#q", tt.path, err, tt.err) + } + } + + if m.Path != tt.m.Path || (tt.m.Version != "" && m.Version != tt.m.Version) { + t.Errorf("queryImport(_, %q) = %v, _; want %v", tt.path, m, tt.m) + } + }) + } +} diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go new file mode 100644 index 0000000..f4f4a68 --- /dev/null +++ b/src/cmd/go/internal/modload/init.go @@ -0,0 +1,2042 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "internal/lazyregexp" + "io" + "os" + "path" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/gover" + "cmd/go/internal/lockedfile" + "cmd/go/internal/modfetch" + "cmd/go/internal/search" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" +) + +// Variables set by other packages. +// +// TODO(#40775): See if these can be plumbed as explicit parameters. +var ( + // RootMode determines whether a module root is needed. + RootMode Root + + // ForceUseModules may be set to force modules to be enabled when + // GO111MODULE=auto or to report an error when GO111MODULE=off. + ForceUseModules bool + + allowMissingModuleImports bool + + // ExplicitWriteGoMod prevents LoadPackages, ListModules, and other functions + // from updating go.mod and go.sum or reporting errors when updates are + // needed. A package should set this if it would cause go.mod to be written + // multiple times (for example, 'go get' calls LoadPackages multiple times) or + // if it needs some other operation to be successful before go.mod and go.sum + // can be written (for example, 'go mod download' must download modules before + // adding sums to go.sum). Packages that set this are responsible for calling + // WriteGoMod explicitly. + ExplicitWriteGoMod bool +) + +// Variables set in Init. +var ( + initialized bool + + // These are primarily used to initialize the MainModules, and should be + // eventually superseded by them but are still used in cases where the module + // roots are required but MainModules hasn't been initialized yet. Set to + // the modRoots of the main modules. + // modRoots != nil implies len(modRoots) > 0 + modRoots []string + gopath string +) + +// EnterModule resets MainModules and requirements to refer to just this one module. +func EnterModule(ctx context.Context, enterModroot string) { + MainModules = nil // reset MainModules + requirements = nil + workFilePath = "" // Force module mode + modfetch.Reset() + + modRoots = []string{enterModroot} + LoadModFile(ctx) +} + +// Variable set in InitWorkfile +var ( + // Set to the path to the go.work file, or "" if workspace mode is disabled. + workFilePath string +) + +type MainModuleSet struct { + // versions are the module.Version values of each of the main modules. + // For each of them, the Path fields are ordinary module paths and the Version + // fields are empty strings. + // versions is clipped (len=cap). + versions []module.Version + + // modRoot maps each module in versions to its absolute filesystem path. + modRoot map[module.Version]string + + // pathPrefix is the path prefix for packages in the module, without a trailing + // slash. For most modules, pathPrefix is just version.Path, but the + // standard-library module "std" has an empty prefix. + pathPrefix map[module.Version]string + + // inGorootSrc caches whether modRoot is within GOROOT/src. + // The "std" module is special within GOROOT/src, but not otherwise. + inGorootSrc map[module.Version]bool + + modFiles map[module.Version]*modfile.File + + modContainingCWD module.Version + + workFile *modfile.WorkFile + + workFileReplaceMap map[module.Version]module.Version + // highest replaced version of each module path; empty string for wildcard-only replacements + highestReplaced map[string]string + + indexMu sync.Mutex + indices map[module.Version]*modFileIndex +} + +func (mms *MainModuleSet) PathPrefix(m module.Version) string { + return mms.pathPrefix[m] +} + +// Versions returns the module.Version values of each of the main modules. +// For each of them, the Path fields are ordinary module paths and the Version +// fields are empty strings. +// Callers should not modify the returned slice. +func (mms *MainModuleSet) Versions() []module.Version { + if mms == nil { + return nil + } + return mms.versions +} + +func (mms *MainModuleSet) Contains(path string) bool { + if mms == nil { + return false + } + for _, v := range mms.versions { + if v.Path == path { + return true + } + } + return false +} + +func (mms *MainModuleSet) ModRoot(m module.Version) string { + if mms == nil { + return "" + } + return mms.modRoot[m] +} + +func (mms *MainModuleSet) InGorootSrc(m module.Version) bool { + if mms == nil { + return false + } + return mms.inGorootSrc[m] +} + +func (mms *MainModuleSet) mustGetSingleMainModule() module.Version { + if mms == nil || len(mms.versions) == 0 { + panic("internal error: mustGetSingleMainModule called in context with no main modules") + } + if len(mms.versions) != 1 { + if inWorkspaceMode() { + panic("internal error: mustGetSingleMainModule called in workspace mode") + } else { + panic("internal error: multiple main modules present outside of workspace mode") + } + } + return mms.versions[0] +} + +func (mms *MainModuleSet) GetSingleIndexOrNil() *modFileIndex { + if mms == nil { + return nil + } + if len(mms.versions) == 0 { + return nil + } + return mms.indices[mms.mustGetSingleMainModule()] +} + +func (mms *MainModuleSet) Index(m module.Version) *modFileIndex { + mms.indexMu.Lock() + defer mms.indexMu.Unlock() + return mms.indices[m] +} + +func (mms *MainModuleSet) SetIndex(m module.Version, index *modFileIndex) { + mms.indexMu.Lock() + defer mms.indexMu.Unlock() + mms.indices[m] = index +} + +func (mms *MainModuleSet) ModFile(m module.Version) *modfile.File { + return mms.modFiles[m] +} + +func (mms *MainModuleSet) WorkFile() *modfile.WorkFile { + return mms.workFile +} + +func (mms *MainModuleSet) Len() int { + if mms == nil { + return 0 + } + return len(mms.versions) +} + +// ModContainingCWD returns the main module containing the working directory, +// or module.Version{} if none of the main modules contain the working +// directory. +func (mms *MainModuleSet) ModContainingCWD() module.Version { + return mms.modContainingCWD +} + +func (mms *MainModuleSet) HighestReplaced() map[string]string { + return mms.highestReplaced +} + +// GoVersion returns the go version set on the single module, in module mode, +// or the go.work file in workspace mode. +func (mms *MainModuleSet) GoVersion() string { + if inWorkspaceMode() { + return gover.FromGoWork(mms.workFile) + } + if mms != nil && len(mms.versions) == 1 { + f := mms.ModFile(mms.mustGetSingleMainModule()) + if f == nil { + // Special case: we are outside a module, like 'go run x.go'. + // Assume the local Go version. + // TODO(#49228): Clean this up; see loadModFile. + return gover.Local() + } + return gover.FromGoMod(f) + } + return gover.DefaultGoModVersion +} + +// Toolchain returns the toolchain set on the single module, in module mode, +// or the go.work file in workspace mode. +func (mms *MainModuleSet) Toolchain() string { + if inWorkspaceMode() { + if mms.workFile != nil && mms.workFile.Toolchain != nil { + return mms.workFile.Toolchain.Name + } + return "go" + mms.GoVersion() + } + if mms != nil && len(mms.versions) == 1 { + f := mms.ModFile(mms.mustGetSingleMainModule()) + if f == nil { + // Special case: we are outside a module, like 'go run x.go'. + // Assume the local Go version. + // TODO(#49228): Clean this up; see loadModFile. + return gover.LocalToolchain() + } + if f.Toolchain != nil { + return f.Toolchain.Name + } + } + return "go" + mms.GoVersion() +} + +func (mms *MainModuleSet) WorkFileReplaceMap() map[module.Version]module.Version { + return mms.workFileReplaceMap +} + +var MainModules *MainModuleSet + +type Root int + +const ( + // AutoRoot is the default for most commands. modload.Init will look for + // a go.mod file in the current directory or any parent. If none is found, + // modules may be disabled (GO111MODULE=auto) or commands may run in a + // limited module mode. + AutoRoot Root = iota + + // NoRoot is used for commands that run in module mode and ignore any go.mod + // file the current directory or in parent directories. + NoRoot + + // NeedRoot is used for commands that must run in module mode and don't + // make sense without a main module. + NeedRoot +) + +// ModFile returns the parsed go.mod file. +// +// Note that after calling LoadPackages or LoadModGraph, +// the require statements in the modfile.File are no longer +// the source of truth and will be ignored: edits made directly +// will be lost at the next call to WriteGoMod. +// To make permanent changes to the require statements +// in go.mod, edit it before loading. +func ModFile() *modfile.File { + Init() + modFile := MainModules.ModFile(MainModules.mustGetSingleMainModule()) + if modFile == nil { + die() + } + return modFile +} + +func BinDir() string { + Init() + if cfg.GOBIN != "" { + return cfg.GOBIN + } + if gopath == "" { + return "" + } + return filepath.Join(gopath, "bin") +} + +// InitWorkfile initializes the workFilePath variable for commands that +// operate in workspace mode. It should not be called by other commands, +// for example 'go mod tidy', that don't operate in workspace mode. +func InitWorkfile() { + workFilePath = FindGoWork(base.Cwd()) +} + +// FindGoWork returns the name of the go.work file for this command, +// or the empty string if there isn't one. +// Most code should use Init and Enabled rather than use this directly. +// It is exported mainly for Go toolchain switching, which must process +// the go.work very early at startup. +func FindGoWork(wd string) string { + if RootMode == NoRoot { + return "" + } + + switch gowork := cfg.Getenv("GOWORK"); gowork { + case "off": + return "" + case "", "auto": + return findWorkspaceFile(wd) + default: + if !filepath.IsAbs(gowork) { + base.Fatalf("go: invalid GOWORK: not an absolute path") + } + return gowork + } +} + +// WorkFilePath returns the absolute path of the go.work file, or "" if not in +// workspace mode. WorkFilePath must be called after InitWorkfile. +func WorkFilePath() string { + return workFilePath +} + +// Reset clears all the initialized, cached state about the use of modules, +// so that we can start over. +func Reset() { + initialized = false + ForceUseModules = false + RootMode = 0 + modRoots = nil + cfg.ModulesEnabled = false + MainModules = nil + requirements = nil + workFilePath = "" + modfetch.Reset() +} + +// Init determines whether module mode is enabled, locates the root of the +// current module (if any), sets environment variables for Git subprocesses, and +// configures the cfg, codehost, load, modfetch, and search packages for use +// with modules. +func Init() { + if initialized { + return + } + initialized = true + + // Keep in sync with WillBeEnabled. We perform extra validation here, and + // there are lots of diagnostics and side effects, so we can't use + // WillBeEnabled directly. + var mustUseModules bool + env := cfg.Getenv("GO111MODULE") + switch env { + default: + base.Fatalf("go: unknown environment setting GO111MODULE=%s", env) + case "auto": + mustUseModules = ForceUseModules + case "on", "": + mustUseModules = true + case "off": + if ForceUseModules { + base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") + } + mustUseModules = false + return + } + + if err := fsys.Init(base.Cwd()); err != nil { + base.Fatal(err) + } + + // Disable any prompting for passwords by Git. + // Only has an effect for 2.3.0 or later, but avoiding + // the prompt in earlier versions is just too hard. + // If user has explicitly set GIT_TERMINAL_PROMPT=1, keep + // prompting. + // See golang.org/issue/9341 and golang.org/issue/12706. + if os.Getenv("GIT_TERMINAL_PROMPT") == "" { + os.Setenv("GIT_TERMINAL_PROMPT", "0") + } + + // Disable any ssh connection pooling by Git. + // If a Git subprocess forks a child into the background to cache a new connection, + // that child keeps stdout/stderr open. After the Git subprocess exits, + // os/exec expects to be able to read from the stdout/stderr pipe + // until EOF to get all the data that the Git subprocess wrote before exiting. + // The EOF doesn't come until the child exits too, because the child + // is holding the write end of the pipe. + // This is unfortunate, but it has come up at least twice + // (see golang.org/issue/13453 and golang.org/issue/16104) + // and confuses users when it does. + // If the user has explicitly set GIT_SSH or GIT_SSH_COMMAND, + // assume they know what they are doing and don't step on it. + // But default to turning off ControlMaster. + if os.Getenv("GIT_SSH") == "" && os.Getenv("GIT_SSH_COMMAND") == "" { + os.Setenv("GIT_SSH_COMMAND", "ssh -o ControlMaster=no -o BatchMode=yes") + } + + if os.Getenv("GCM_INTERACTIVE") == "" { + os.Setenv("GCM_INTERACTIVE", "never") + } + if modRoots != nil { + // modRoot set before Init was called ("go mod init" does this). + // No need to search for go.mod. + } else if RootMode == NoRoot { + if cfg.ModFile != "" && !base.InGOFLAGS("-modfile") { + base.Fatalf("go: -modfile cannot be used with commands that ignore the current module") + } + modRoots = nil + } else if workFilePath != "" { + // We're in workspace mode, which implies module mode. + if cfg.ModFile != "" { + base.Fatalf("go: -modfile cannot be used in workspace mode") + } + } else { + if modRoot := findModuleRoot(base.Cwd()); modRoot == "" { + if cfg.ModFile != "" { + base.Fatalf("go: cannot find main module, but -modfile was set.\n\t-modfile cannot be used to set the module root directory.") + } + if RootMode == NeedRoot { + base.Fatal(ErrNoModRoot) + } + if !mustUseModules { + // GO111MODULE is 'auto', and we can't find a module root. + // Stay in GOPATH mode. + return + } + } else if search.InDir(modRoot, os.TempDir()) == "." { + // If you create /tmp/go.mod for experimenting, + // then any tests that create work directories under /tmp + // will find it and get modules when they're not expecting them. + // It's a bit of a peculiar thing to disallow but quite mysterious + // when it happens. See golang.org/issue/26708. + fmt.Fprintf(os.Stderr, "go: warning: ignoring go.mod in system temp root %v\n", os.TempDir()) + if RootMode == NeedRoot { + base.Fatal(ErrNoModRoot) + } + if !mustUseModules { + return + } + } else { + modRoots = []string{modRoot} + } + } + if cfg.ModFile != "" && !strings.HasSuffix(cfg.ModFile, ".mod") { + base.Fatalf("go: -modfile=%s: file does not have .mod extension", cfg.ModFile) + } + + // We're in module mode. Set any global variables that need to be set. + cfg.ModulesEnabled = true + setDefaultBuildMod() + list := filepath.SplitList(cfg.BuildContext.GOPATH) + if len(list) > 0 && list[0] != "" { + gopath = list[0] + if _, err := fsys.Stat(filepath.Join(gopath, "go.mod")); err == nil { + fmt.Fprintf(os.Stderr, "go: warning: ignoring go.mod in $GOPATH %v\n", gopath) + if RootMode == NeedRoot { + base.Fatal(ErrNoModRoot) + } + if !mustUseModules { + return + } + } + } +} + +// WillBeEnabled checks whether modules should be enabled but does not +// initialize modules by installing hooks. If Init has already been called, +// WillBeEnabled returns the same result as Enabled. +// +// This function is needed to break a cycle. The main package needs to know +// whether modules are enabled in order to install the module or GOPATH version +// of 'go get', but Init reads the -modfile flag in 'go get', so it shouldn't +// be called until the command is installed and flags are parsed. Instead of +// calling Init and Enabled, the main package can call this function. +func WillBeEnabled() bool { + if modRoots != nil || cfg.ModulesEnabled { + // Already enabled. + return true + } + if initialized { + // Initialized, not enabled. + return false + } + + // Keep in sync with Init. Init does extra validation and prints warnings or + // exits, so it can't call this function directly. + env := cfg.Getenv("GO111MODULE") + switch env { + case "on", "": + return true + case "auto": + break + default: + return false + } + + return FindGoMod(base.Cwd()) != "" +} + +// FindGoMod returns the name of the go.mod file for this command, +// or the empty string if there isn't one. +// Most code should use Init and Enabled rather than use this directly. +// It is exported mainly for Go toolchain switching, which must process +// the go.mod very early at startup. +func FindGoMod(wd string) string { + modRoot := findModuleRoot(wd) + if modRoot == "" { + // GO111MODULE is 'auto', and we can't find a module root. + // Stay in GOPATH mode. + return "" + } + if search.InDir(modRoot, os.TempDir()) == "." { + // If you create /tmp/go.mod for experimenting, + // then any tests that create work directories under /tmp + // will find it and get modules when they're not expecting them. + // It's a bit of a peculiar thing to disallow but quite mysterious + // when it happens. See golang.org/issue/26708. + return "" + } + return filepath.Join(modRoot, "go.mod") +} + +// Enabled reports whether modules are (or must be) enabled. +// If modules are enabled but there is no main module, Enabled returns true +// and then the first use of module information will call die +// (usually through MustModRoot). +func Enabled() bool { + Init() + return modRoots != nil || cfg.ModulesEnabled +} + +func VendorDir() string { + if inWorkspaceMode() { + return filepath.Join(filepath.Dir(WorkFilePath()), "vendor") + } + // Even if -mod=vendor, we could be operating with no mod root (and thus no + // vendor directory). As long as there are no dependencies that is expected + // to work. See script/vendor_outside_module.txt. + modRoot := MainModules.ModRoot(MainModules.mustGetSingleMainModule()) + if modRoot == "" { + panic("vendor directory does not exist when in single module mode outside of a module") + } + return filepath.Join(modRoot, "vendor") +} + +func inWorkspaceMode() bool { + if !initialized { + panic("inWorkspaceMode called before modload.Init called") + } + if !Enabled() { + return false + } + return workFilePath != "" +} + +// HasModRoot reports whether a main module is present. +// HasModRoot may return false even if Enabled returns true: for example, 'get' +// does not require a main module. +func HasModRoot() bool { + Init() + return modRoots != nil +} + +// MustHaveModRoot checks that a main module or main modules are present, +// and calls base.Fatalf if there are no main modules. +func MustHaveModRoot() { + Init() + if !HasModRoot() { + die() + } +} + +// ModFilePath returns the path that would be used for the go.mod +// file, if in module mode. ModFilePath calls base.Fatalf if there is no main +// module, even if -modfile is set. +func ModFilePath() string { + MustHaveModRoot() + return modFilePath(findModuleRoot(base.Cwd())) +} + +func modFilePath(modRoot string) string { + if cfg.ModFile != "" { + return cfg.ModFile + } + return filepath.Join(modRoot, "go.mod") +} + +func die() { + if cfg.Getenv("GO111MODULE") == "off" { + base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") + } + if inWorkspaceMode() { + base.Fatalf("go: no modules were found in the current workspace; see 'go help work'") + } + if dir, name := findAltConfig(base.Cwd()); dir != "" { + rel, err := filepath.Rel(base.Cwd(), dir) + if err != nil { + rel = dir + } + cdCmd := "" + if rel != "." { + cdCmd = fmt.Sprintf("cd %s && ", rel) + } + base.Fatalf("go: cannot find main module, but found %s in %s\n\tto create a module there, run:\n\t%sgo mod init", name, dir, cdCmd) + } + base.Fatal(ErrNoModRoot) +} + +var ErrNoModRoot = errors.New("go.mod file not found in current directory or any parent directory; see 'go help modules'") + +type goModDirtyError struct{} + +func (goModDirtyError) Error() string { + if cfg.BuildModExplicit { + return fmt.Sprintf("updates to go.mod needed, disabled by -mod=%v; to update it:\n\tgo mod tidy", cfg.BuildMod) + } + if cfg.BuildModReason != "" { + return fmt.Sprintf("updates to go.mod needed, disabled by -mod=%s\n\t(%s)\n\tto update it:\n\tgo mod tidy", cfg.BuildMod, cfg.BuildModReason) + } + return "updates to go.mod needed; to update it:\n\tgo mod tidy" +} + +var errGoModDirty error = goModDirtyError{} + +func loadWorkFile(path string) (workFile *modfile.WorkFile, modRoots []string, err error) { + workDir := filepath.Dir(path) + wf, err := ReadWorkFile(path) + if err != nil { + return nil, nil, err + } + seen := map[string]bool{} + for _, d := range wf.Use { + modRoot := d.Path + if !filepath.IsAbs(modRoot) { + modRoot = filepath.Join(workDir, modRoot) + } + + if seen[modRoot] { + return nil, nil, fmt.Errorf("path %s appears multiple times in workspace", modRoot) + } + seen[modRoot] = true + modRoots = append(modRoots, modRoot) + } + + return wf, modRoots, nil +} + +// ReadWorkFile reads and parses the go.work file at the given path. +func ReadWorkFile(path string) (*modfile.WorkFile, error) { + workData, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + f, err := modfile.ParseWork(path, workData, nil) + if err != nil { + return nil, err + } + if f.Go != nil && gover.Compare(f.Go.Version, gover.Local()) > 0 && cfg.CmdName != "work edit" { + base.Fatal(&gover.TooNewError{What: base.ShortPath(path), GoVersion: f.Go.Version}) + } + return f, nil +} + +// WriteWorkFile cleans and writes out the go.work file to the given path. +func WriteWorkFile(path string, wf *modfile.WorkFile) error { + wf.SortBlocks() + wf.Cleanup() + out := modfile.Format(wf.Syntax) + + return os.WriteFile(path, out, 0666) +} + +// UpdateWorkGoVersion updates the go line in wf to be at least goVers, +// reporting whether it changed the file. +func UpdateWorkGoVersion(wf *modfile.WorkFile, goVers string) (changed bool) { + old := gover.FromGoWork(wf) + if gover.Compare(old, goVers) >= 0 { + return false + } + + wf.AddGoStmt(goVers) + + // We wrote a new go line. For reproducibility, + // if the toolchain running right now is newer than the new toolchain line, + // update the toolchain line to record the newer toolchain. + // The user never sets the toolchain explicitly in a 'go work' command, + // so this is only happening as a result of a go or toolchain line found + // in a module. + // If the toolchain running right now is a dev toolchain (like "go1.21") + // writing 'toolchain go1.21' will not be useful, since that's not an actual + // toolchain you can download and run. In that case fall back to at least + // checking that the toolchain is new enough for the Go version. + toolchain := "go" + old + if wf.Toolchain != nil { + toolchain = wf.Toolchain.Name + } + if gover.IsLang(gover.Local()) { + toolchain = gover.ToolchainMax(toolchain, "go"+goVers) + } else { + toolchain = gover.ToolchainMax(toolchain, "go"+gover.Local()) + } + + // Drop the toolchain line if it is implied by the go line + // or if it is asking for a toolchain older than Go 1.21, + // which will not understand the toolchain line. + if toolchain == "go"+goVers || gover.Compare(gover.FromToolchain(toolchain), gover.GoStrictVersion) < 0 { + wf.DropToolchainStmt() + } else { + wf.AddToolchainStmt(toolchain) + } + return true +} + +// UpdateWorkFile updates comments on directory directives in the go.work +// file to include the associated module path. +func UpdateWorkFile(wf *modfile.WorkFile) { + missingModulePaths := map[string]string{} // module directory listed in file -> abspath modroot + + for _, d := range wf.Use { + if d.Path == "" { + continue // d is marked for deletion. + } + modRoot := d.Path + if d.ModulePath == "" { + missingModulePaths[d.Path] = modRoot + } + } + + // Clean up and annotate directories. + // TODO(matloob): update x/mod to actually add module paths. + for moddir, absmodroot := range missingModulePaths { + _, f, err := ReadModFile(filepath.Join(absmodroot, "go.mod"), nil) + if err != nil { + continue // Error will be reported if modules are loaded. + } + wf.AddUse(moddir, f.Module.Mod.Path) + } +} + +// LoadModFile sets Target and, if there is a main module, parses the initial +// build list from its go.mod file. +// +// LoadModFile may make changes in memory, like adding a go directive and +// ensuring requirements are consistent. The caller is responsible for ensuring +// those changes are written to disk by calling LoadPackages or ListModules +// (unless ExplicitWriteGoMod is set) or by calling WriteGoMod directly. +// +// As a side-effect, LoadModFile may change cfg.BuildMod to "vendor" if +// -mod wasn't set explicitly and automatic vendoring should be enabled. +// +// If LoadModFile or CreateModFile has already been called, LoadModFile returns +// the existing in-memory requirements (rather than re-reading them from disk). +// +// LoadModFile checks the roots of the module graph for consistency with each +// other, but unlike LoadModGraph does not load the full module graph or check +// it for global consistency. Most callers outside of the modload package should +// use LoadModGraph instead. +func LoadModFile(ctx context.Context) *Requirements { + rs, err := loadModFile(ctx, nil) + if err != nil { + base.Fatal(err) + } + return rs +} + +func loadModFile(ctx context.Context, opts *PackageOpts) (*Requirements, error) { + if requirements != nil { + return requirements, nil + } + + Init() + var workFile *modfile.WorkFile + if inWorkspaceMode() { + var err error + workFile, modRoots, err = loadWorkFile(workFilePath) + if err != nil { + return nil, fmt.Errorf("reading go.work: %w", err) + } + for _, modRoot := range modRoots { + sumFile := strings.TrimSuffix(modFilePath(modRoot), ".mod") + ".sum" + modfetch.WorkspaceGoSumFiles = append(modfetch.WorkspaceGoSumFiles, sumFile) + } + modfetch.GoSumFile = workFilePath + ".sum" + } else if len(modRoots) == 0 { + // We're in module mode, but not inside a module. + // + // Commands like 'go build', 'go run', 'go list' have no go.mod file to + // read or write. They would need to find and download the latest versions + // of a potentially large number of modules with no way to save version + // information. We can succeed slowly (but not reproducibly), but that's + // not usually a good experience. + // + // Instead, we forbid resolving import paths to modules other than std and + // cmd. Users may still build packages specified with .go files on the + // command line, but they'll see an error if those files import anything + // outside std. + // + // This can be overridden by calling AllowMissingModuleImports. + // For example, 'go get' does this, since it is expected to resolve paths. + // + // See golang.org/issue/32027. + } else { + modfetch.GoSumFile = strings.TrimSuffix(modFilePath(modRoots[0]), ".mod") + ".sum" + } + if len(modRoots) == 0 { + // TODO(#49228): Instead of creating a fake module with an empty modroot, + // make MainModules.Len() == 0 mean that we're in module mode but not inside + // any module. + mainModule := module.Version{Path: "command-line-arguments"} + MainModules = makeMainModules([]module.Version{mainModule}, []string{""}, []*modfile.File{nil}, []*modFileIndex{nil}, nil) + var ( + goVersion string + pruning modPruning + roots []module.Version + direct = map[string]bool{"go": true} + ) + if inWorkspaceMode() { + // Since we are in a workspace, the Go version for the synthetic + // "command-line-arguments" module must not exceed the Go version + // for the workspace. + goVersion = MainModules.GoVersion() + pruning = workspace + roots = []module.Version{ + mainModule, + {Path: "go", Version: goVersion}, + {Path: "toolchain", Version: gover.LocalToolchain()}, + } + } else { + goVersion = gover.Local() + pruning = pruningForGoVersion(goVersion) + roots = []module.Version{ + {Path: "go", Version: goVersion}, + {Path: "toolchain", Version: gover.LocalToolchain()}, + } + } + rawGoVersion.Store(mainModule, goVersion) + requirements = newRequirements(pruning, roots, direct) + if cfg.BuildMod == "vendor" { + // For issue 56536: Some users may have GOFLAGS=-mod=vendor set. + // Make sure it behaves as though the fake module is vendored + // with no dependencies. + requirements.initVendor(nil) + } + return requirements, nil + } + + var modFiles []*modfile.File + var mainModules []module.Version + var indices []*modFileIndex + var errs []error + for _, modroot := range modRoots { + gomod := modFilePath(modroot) + var fixed bool + data, f, err := ReadModFile(gomod, fixVersion(ctx, &fixed)) + if err != nil { + if inWorkspaceMode() { + if tooNew, ok := err.(*gover.TooNewError); ok && !strings.HasPrefix(cfg.CmdName, "work ") { + // Switching to a newer toolchain won't help - the go.work has the wrong version. + // Report this more specific error, unless we are a command like 'go work use' + // or 'go work sync', which will fix the problem after the caller sees the TooNewError + // and switches to a newer toolchain. + err = errWorkTooOld(gomod, workFile, tooNew.GoVersion) + } else { + err = fmt.Errorf("cannot load module %s listed in go.work file: %w", + base.ShortPath(filepath.Dir(gomod)), err) + } + } + errs = append(errs, err) + continue + } + if inWorkspaceMode() && !strings.HasPrefix(cfg.CmdName, "work ") { + // Refuse to use workspace if its go version is too old. + // Disable this check if we are a workspace command like work use or work sync, + // which will fix the problem. + mv := gover.FromGoMod(f) + wv := gover.FromGoWork(workFile) + if gover.Compare(mv, wv) > 0 && gover.Compare(mv, gover.GoStrictVersion) >= 0 { + errs = append(errs, errWorkTooOld(gomod, workFile, mv)) + continue + } + } + + modFiles = append(modFiles, f) + mainModule := f.Module.Mod + mainModules = append(mainModules, mainModule) + indices = append(indices, indexModFile(data, f, mainModule, fixed)) + + if err := module.CheckImportPath(f.Module.Mod.Path); err != nil { + if pathErr, ok := err.(*module.InvalidPathError); ok { + pathErr.Kind = "module" + } + errs = append(errs, err) + } + } + if len(errs) > 0 { + return nil, errors.Join(errs...) + } + + MainModules = makeMainModules(mainModules, modRoots, modFiles, indices, workFile) + setDefaultBuildMod() // possibly enable automatic vendoring + rs := requirementsFromModFiles(ctx, workFile, modFiles, opts) + + if cfg.BuildMod == "vendor" { + readVendorList(VendorDir()) + var indexes []*modFileIndex + var modFiles []*modfile.File + var modRoots []string + for _, m := range MainModules.Versions() { + indexes = append(indexes, MainModules.Index(m)) + modFiles = append(modFiles, MainModules.ModFile(m)) + modRoots = append(modRoots, MainModules.ModRoot(m)) + } + checkVendorConsistency(indexes, modFiles, modRoots) + rs.initVendor(vendorList) + } + + if inWorkspaceMode() { + // We don't need to update the mod file so return early. + requirements = rs + return rs, nil + } + + mainModule := MainModules.mustGetSingleMainModule() + + if rs.hasRedundantRoot() { + // If any module path appears more than once in the roots, we know that the + // go.mod file needs to be updated even though we have not yet loaded any + // transitive dependencies. + var err error + rs, err = updateRoots(ctx, rs.direct, rs, nil, nil, false) + if err != nil { + return nil, err + } + } + + if MainModules.Index(mainModule).goVersion == "" && rs.pruning != workspace { + // TODO(#45551): Do something more principled instead of checking + // cfg.CmdName directly here. + if cfg.BuildMod == "mod" && cfg.CmdName != "mod graph" && cfg.CmdName != "mod why" { + // go line is missing from go.mod; add one there and add to derived requirements. + v := gover.Local() + if opts != nil && opts.TidyGoVersion != "" { + v = opts.TidyGoVersion + } + addGoStmt(MainModules.ModFile(mainModule), mainModule, v) + rs = overrideRoots(ctx, rs, []module.Version{{Path: "go", Version: v}}) + + // We need to add a 'go' version to the go.mod file, but we must assume + // that its existing contents match something between Go 1.11 and 1.16. + // Go 1.11 through 1.16 do not support graph pruning, but the latest Go + // version uses a pruned module graph — so we need to convert the + // requirements to support pruning. + if gover.Compare(v, gover.ExplicitIndirectVersion) >= 0 { + var err error + rs, err = convertPruning(ctx, rs, pruned) + if err != nil { + return nil, err + } + } + } else { + rawGoVersion.Store(mainModule, gover.DefaultGoModVersion) + } + } + + requirements = rs + return requirements, nil +} + +func errWorkTooOld(gomod string, wf *modfile.WorkFile, goVers string) error { + return fmt.Errorf("module %s listed in go.work file requires go >= %s, but go.work lists go %s; to update it:\n\tgo work use", + base.ShortPath(filepath.Dir(gomod)), goVers, gover.FromGoWork(wf)) +} + +// CreateModFile initializes a new module by creating a go.mod file. +// +// If modPath is empty, CreateModFile will attempt to infer the path from the +// directory location within GOPATH. +// +// If a vendoring configuration file is present, CreateModFile will attempt to +// translate it to go.mod directives. The resulting build list may not be +// exactly the same as in the legacy configuration (for example, we can't get +// packages at multiple versions from the same module). +func CreateModFile(ctx context.Context, modPath string) { + modRoot := base.Cwd() + modRoots = []string{modRoot} + Init() + modFilePath := modFilePath(modRoot) + if _, err := fsys.Stat(modFilePath); err == nil { + base.Fatalf("go: %s already exists", modFilePath) + } + + if modPath == "" { + var err error + modPath, err = findModulePath(modRoot) + if err != nil { + base.Fatal(err) + } + } else if err := module.CheckImportPath(modPath); err != nil { + if pathErr, ok := err.(*module.InvalidPathError); ok { + pathErr.Kind = "module" + // Same as build.IsLocalPath() + if pathErr.Path == "." || pathErr.Path == ".." || + strings.HasPrefix(pathErr.Path, "./") || strings.HasPrefix(pathErr.Path, "../") { + pathErr.Err = errors.New("is a local import path") + } + } + base.Fatal(err) + } else if _, _, ok := module.SplitPathVersion(modPath); !ok { + if strings.HasPrefix(modPath, "gopkg.in/") { + invalidMajorVersionMsg := fmt.Errorf("module paths beginning with gopkg.in/ must always have a major version suffix in the form of .vN:\n\tgo mod init %s", suggestGopkgIn(modPath)) + base.Fatalf(`go: invalid module path "%v": %v`, modPath, invalidMajorVersionMsg) + } + invalidMajorVersionMsg := fmt.Errorf("major version suffixes must be in the form of /vN and are only allowed for v2 or later:\n\tgo mod init %s", suggestModulePath(modPath)) + base.Fatalf(`go: invalid module path "%v": %v`, modPath, invalidMajorVersionMsg) + } + + fmt.Fprintf(os.Stderr, "go: creating new go.mod: module %s\n", modPath) + modFile := new(modfile.File) + modFile.AddModuleStmt(modPath) + MainModules = makeMainModules([]module.Version{modFile.Module.Mod}, []string{modRoot}, []*modfile.File{modFile}, []*modFileIndex{nil}, nil) + addGoStmt(modFile, modFile.Module.Mod, gover.Local()) // Add the go directive before converted module requirements. + + rs := requirementsFromModFiles(ctx, nil, []*modfile.File{modFile}, nil) + rs, err := updateRoots(ctx, rs.direct, rs, nil, nil, false) + if err != nil { + base.Fatal(err) + } + requirements = rs + if err := commitRequirements(ctx, WriteOpts{}); err != nil { + base.Fatal(err) + } + + // Suggest running 'go mod tidy' unless the project is empty. Even if we + // imported all the correct requirements above, we're probably missing + // some sums, so the next build command in -mod=readonly will likely fail. + // + // We look for non-hidden .go files or subdirectories to determine whether + // this is an existing project. Walking the tree for packages would be more + // accurate, but could take much longer. + empty := true + files, _ := os.ReadDir(modRoot) + for _, f := range files { + name := f.Name() + if strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") { + continue + } + if strings.HasSuffix(name, ".go") || f.IsDir() { + empty = false + break + } + } + if !empty { + fmt.Fprintf(os.Stderr, "go: to add module requirements and sums:\n\tgo mod tidy\n") + } +} + +// fixVersion returns a modfile.VersionFixer implemented using the Query function. +// +// It resolves commit hashes and branch names to versions, +// canonicalizes versions that appeared in early vgo drafts, +// and does nothing for versions that already appear to be canonical. +// +// The VersionFixer sets 'fixed' if it ever returns a non-canonical version. +func fixVersion(ctx context.Context, fixed *bool) modfile.VersionFixer { + return func(path, vers string) (resolved string, err error) { + defer func() { + if err == nil && resolved != vers { + *fixed = true + } + }() + + // Special case: remove the old -gopkgin- hack. + if strings.HasPrefix(path, "gopkg.in/") && strings.Contains(vers, "-gopkgin-") { + vers = vers[strings.Index(vers, "-gopkgin-")+len("-gopkgin-"):] + } + + // fixVersion is called speculatively on every + // module, version pair from every go.mod file. + // Avoid the query if it looks OK. + _, pathMajor, ok := module.SplitPathVersion(path) + if !ok { + return "", &module.ModuleError{ + Path: path, + Err: &module.InvalidVersionError{ + Version: vers, + Err: fmt.Errorf("malformed module path %q", path), + }, + } + } + if vers != "" && module.CanonicalVersion(vers) == vers { + if err := module.CheckPathMajor(vers, pathMajor); err != nil { + return "", module.VersionError(module.Version{Path: path, Version: vers}, err) + } + return vers, nil + } + + info, err := Query(ctx, path, vers, "", nil) + if err != nil { + return "", err + } + return info.Version, nil + } +} + +// AllowMissingModuleImports allows import paths to be resolved to modules +// when there is no module root. Normally, this is forbidden because it's slow +// and there's no way to make the result reproducible, but some commands +// like 'go get' are expected to do this. +// +// This function affects the default cfg.BuildMod when outside of a module, +// so it can only be called prior to Init. +func AllowMissingModuleImports() { + if initialized { + panic("AllowMissingModuleImports after Init") + } + allowMissingModuleImports = true +} + +// makeMainModules creates a MainModuleSet and associated variables according to +// the given main modules. +func makeMainModules(ms []module.Version, rootDirs []string, modFiles []*modfile.File, indices []*modFileIndex, workFile *modfile.WorkFile) *MainModuleSet { + for _, m := range ms { + if m.Version != "" { + panic("mainModulesCalled with module.Version with non empty Version field: " + fmt.Sprintf("%#v", m)) + } + } + modRootContainingCWD := findModuleRoot(base.Cwd()) + mainModules := &MainModuleSet{ + versions: slices.Clip(ms), + inGorootSrc: map[module.Version]bool{}, + pathPrefix: map[module.Version]string{}, + modRoot: map[module.Version]string{}, + modFiles: map[module.Version]*modfile.File{}, + indices: map[module.Version]*modFileIndex{}, + highestReplaced: map[string]string{}, + workFile: workFile, + } + var workFileReplaces []*modfile.Replace + if workFile != nil { + workFileReplaces = workFile.Replace + mainModules.workFileReplaceMap = toReplaceMap(workFile.Replace) + } + mainModulePaths := make(map[string]bool) + for _, m := range ms { + if mainModulePaths[m.Path] { + base.Errorf("go: module %s appears multiple times in workspace", m.Path) + } + mainModulePaths[m.Path] = true + } + replacedByWorkFile := make(map[string]bool) + replacements := make(map[module.Version]module.Version) + for _, r := range workFileReplaces { + if mainModulePaths[r.Old.Path] && r.Old.Version == "" { + base.Errorf("go: workspace module %v is replaced at all versions in the go.work file. To fix, remove the replacement from the go.work file or specify the version at which to replace the module.", r.Old.Path) + } + replacedByWorkFile[r.Old.Path] = true + v, ok := mainModules.highestReplaced[r.Old.Path] + if !ok || gover.ModCompare(r.Old.Path, r.Old.Version, v) > 0 { + mainModules.highestReplaced[r.Old.Path] = r.Old.Version + } + replacements[r.Old] = r.New + } + for i, m := range ms { + mainModules.pathPrefix[m] = m.Path + mainModules.modRoot[m] = rootDirs[i] + mainModules.modFiles[m] = modFiles[i] + mainModules.indices[m] = indices[i] + + if mainModules.modRoot[m] == modRootContainingCWD { + mainModules.modContainingCWD = m + } + + if rel := search.InDir(rootDirs[i], cfg.GOROOTsrc); rel != "" { + mainModules.inGorootSrc[m] = true + if m.Path == "std" { + // The "std" module in GOROOT/src is the Go standard library. Unlike other + // modules, the packages in the "std" module have no import-path prefix. + // + // Modules named "std" outside of GOROOT/src do not receive this special + // treatment, so it is possible to run 'go test .' in other GOROOTs to + // test individual packages using a combination of the modified package + // and the ordinary standard library. + // (See https://golang.org/issue/30756.) + mainModules.pathPrefix[m] = "" + } + } + + if modFiles[i] != nil { + curModuleReplaces := make(map[module.Version]bool) + for _, r := range modFiles[i].Replace { + if replacedByWorkFile[r.Old.Path] { + continue + } + var newV module.Version = r.New + if WorkFilePath() != "" && newV.Version == "" && !filepath.IsAbs(newV.Path) { + // Since we are in a workspace, we may be loading replacements from + // multiple go.mod files. Relative paths in those replacement are + // relative to the go.mod file, not the workspace, so the same string + // may refer to two different paths and different strings may refer to + // the same path. Convert them all to be absolute instead. + // + // (We could do this outside of a workspace too, but it would mean that + // replacement paths in error strings needlessly differ from what's in + // the go.mod file.) + newV.Path = filepath.Join(rootDirs[i], newV.Path) + } + if prev, ok := replacements[r.Old]; ok && !curModuleReplaces[r.Old] && prev != newV { + base.Fatalf("go: conflicting replacements for %v:\n\t%v\n\t%v\nuse \"go work edit -replace %v=[override]\" to resolve", r.Old, prev, newV, r.Old) + } + curModuleReplaces[r.Old] = true + replacements[r.Old] = newV + + v, ok := mainModules.highestReplaced[r.Old.Path] + if !ok || gover.ModCompare(r.Old.Path, r.Old.Version, v) > 0 { + mainModules.highestReplaced[r.Old.Path] = r.Old.Version + } + } + } + } + return mainModules +} + +// requirementsFromModFiles returns the set of non-excluded requirements from +// the global modFile. +func requirementsFromModFiles(ctx context.Context, workFile *modfile.WorkFile, modFiles []*modfile.File, opts *PackageOpts) *Requirements { + var roots []module.Version + direct := map[string]bool{} + var pruning modPruning + if inWorkspaceMode() { + pruning = workspace + roots = make([]module.Version, len(MainModules.Versions()), 2+len(MainModules.Versions())) + copy(roots, MainModules.Versions()) + goVersion := gover.FromGoWork(workFile) + var toolchain string + if workFile.Toolchain != nil { + toolchain = workFile.Toolchain.Name + } + roots = appendGoAndToolchainRoots(roots, goVersion, toolchain, direct) + } else { + pruning = pruningForGoVersion(MainModules.GoVersion()) + if len(modFiles) != 1 { + panic(fmt.Errorf("requirementsFromModFiles called with %v modfiles outside workspace mode", len(modFiles))) + } + modFile := modFiles[0] + roots, direct = rootsFromModFile(MainModules.mustGetSingleMainModule(), modFile, withToolchainRoot) + } + + gover.ModSort(roots) + rs := newRequirements(pruning, roots, direct) + return rs +} + +type addToolchainRoot bool + +const ( + omitToolchainRoot addToolchainRoot = false + withToolchainRoot = true +) + +func rootsFromModFile(m module.Version, modFile *modfile.File, addToolchainRoot addToolchainRoot) (roots []module.Version, direct map[string]bool) { + direct = make(map[string]bool) + padding := 2 // Add padding for the toolchain and go version, added upon return. + if !addToolchainRoot { + padding = 1 + } + roots = make([]module.Version, 0, padding+len(modFile.Require)) + for _, r := range modFile.Require { + if index := MainModules.Index(m); index != nil && index.exclude[r.Mod] { + if cfg.BuildMod == "mod" { + fmt.Fprintf(os.Stderr, "go: dropping requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + } else { + fmt.Fprintf(os.Stderr, "go: ignoring requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + } + continue + } + + roots = append(roots, r.Mod) + if !r.Indirect { + direct[r.Mod.Path] = true + } + } + goVersion := gover.FromGoMod(modFile) + var toolchain string + if addToolchainRoot && modFile.Toolchain != nil { + toolchain = modFile.Toolchain.Name + } + roots = appendGoAndToolchainRoots(roots, goVersion, toolchain, direct) + return roots, direct +} + +func appendGoAndToolchainRoots(roots []module.Version, goVersion, toolchain string, direct map[string]bool) []module.Version { + // Add explicit go and toolchain versions, inferring as needed. + roots = append(roots, module.Version{Path: "go", Version: goVersion}) + direct["go"] = true // Every module directly uses the language and runtime. + + if toolchain != "" { + roots = append(roots, module.Version{Path: "toolchain", Version: toolchain}) + // Leave the toolchain as indirect: nothing in the user's module directly + // imports a package from the toolchain, and (like an indirect dependency in + // a module without graph pruning) we may remove the toolchain line + // automatically if the 'go' version is changed so that it implies the exact + // same toolchain. + } + return roots +} + +// setDefaultBuildMod sets a default value for cfg.BuildMod if the -mod flag +// wasn't provided. setDefaultBuildMod may be called multiple times. +func setDefaultBuildMod() { + if cfg.BuildModExplicit { + if inWorkspaceMode() && cfg.BuildMod != "readonly" && cfg.BuildMod != "vendor" { + base.Fatalf("go: -mod may only be set to readonly or vendor when in workspace mode, but it is set to %q"+ + "\n\tRemove the -mod flag to use the default readonly value, "+ + "\n\tor set GOWORK=off to disable workspace mode.", cfg.BuildMod) + } + // Don't override an explicit '-mod=' argument. + return + } + + // TODO(#40775): commands should pass in the module mode as an option + // to modload functions instead of relying on an implicit setting + // based on command name. + switch cfg.CmdName { + case "get", "mod download", "mod init", "mod tidy", "work sync": + // These commands are intended to update go.mod and go.sum. + cfg.BuildMod = "mod" + return + case "mod graph", "mod verify", "mod why": + // These commands should not update go.mod or go.sum, but they should be + // able to fetch modules not in go.sum and should not report errors if + // go.mod is inconsistent. They're useful for debugging, and they need + // to work in buggy situations. + cfg.BuildMod = "mod" + return + case "mod vendor", "work vendor": + cfg.BuildMod = "readonly" + return + } + if modRoots == nil { + if allowMissingModuleImports { + cfg.BuildMod = "mod" + } else { + cfg.BuildMod = "readonly" + } + return + } + + if len(modRoots) >= 1 { + var goVersion string + var versionSource string + if inWorkspaceMode() { + versionSource = "go.work" + if wfg := MainModules.WorkFile().Go; wfg != nil { + goVersion = wfg.Version + } + } else { + versionSource = "go.mod" + index := MainModules.GetSingleIndexOrNil() + if index != nil { + goVersion = index.goVersion + } + } + vendorDir := "" + if workFilePath != "" { + vendorDir = filepath.Join(filepath.Dir(workFilePath), "vendor") + } else { + if len(modRoots) != 1 { + panic(fmt.Errorf("outside workspace mode, but have %v modRoots", modRoots)) + } + vendorDir = filepath.Join(modRoots[0], "vendor") + } + if fi, err := fsys.Stat(vendorDir); err == nil && fi.IsDir() { + modGo := "unspecified" + if goVersion != "" { + if gover.Compare(goVersion, "1.14") < 0 { + // The go version is less than 1.14. Don't set -mod=vendor by default. + // Since a vendor directory exists, we should record why we didn't use it. + // This message won't normally be shown, but it may appear with import errors. + cfg.BuildModReason = fmt.Sprintf("Go version in "+versionSource+" is %s, so vendor directory was not used.", modGo) + } else { + vendoredWorkspace, err := modulesTextIsForWorkspace(vendorDir) + if err != nil { + base.Fatalf("go: reading modules.txt for vendor directory: %v", err) + } + if vendoredWorkspace != (versionSource == "go.work") { + if vendoredWorkspace { + cfg.BuildModReason = "Outside workspace mode, but vendor directory is for a workspace." + } else { + cfg.BuildModReason = "In workspace mode, but vendor directory is not for a workspace" + } + } else { + // The Go version is at least 1.14, a vendor directory exists, and + // the modules.txt was generated in the same mode the command is running in. + // Set -mod=vendor by default. + cfg.BuildMod = "vendor" + cfg.BuildModReason = "Go version in " + versionSource + " is at least 1.14 and vendor directory exists." + return + } + } + modGo = goVersion + } + + } + } + + cfg.BuildMod = "readonly" +} + +func modulesTextIsForWorkspace(vendorDir string) (bool, error) { + f, err := fsys.Open(filepath.Join(vendorDir, "modules.txt")) + if errors.Is(err, os.ErrNotExist) { + // Some vendor directories exist that don't contain modules.txt. + // This mostly happens when converting to modules. + // We want to preserve the behavior that mod=vendor is set (even though + // readVendorList does nothing in that case). + return false, nil + } + if err != nil { + return false, err + } + var buf [512]byte + n, err := f.Read(buf[:]) + if err != nil && err != io.EOF { + return false, err + } + line, _, _ := strings.Cut(string(buf[:n]), "\n") + if annotations, ok := strings.CutPrefix(line, "## "); ok { + for _, entry := range strings.Split(annotations, ";") { + entry = strings.TrimSpace(entry) + if entry == "workspace" { + return true, nil + } + } + } + return false, nil +} + +func mustHaveCompleteRequirements() bool { + return cfg.BuildMod != "mod" && !inWorkspaceMode() +} + +// addGoStmt adds a go directive to the go.mod file if it does not already +// include one. The 'go' version added, if any, is the latest version supported +// by this toolchain. +func addGoStmt(modFile *modfile.File, mod module.Version, v string) { + if modFile.Go != nil && modFile.Go.Version != "" { + return + } + forceGoStmt(modFile, mod, v) +} + +func forceGoStmt(modFile *modfile.File, mod module.Version, v string) { + if err := modFile.AddGoStmt(v); err != nil { + base.Fatalf("go: internal error: %v", err) + } + rawGoVersion.Store(mod, v) +} + +var altConfigs = []string{ + ".git/config", +} + +func findModuleRoot(dir string) (roots string) { + if dir == "" { + panic("dir not set") + } + dir = filepath.Clean(dir) + + // Look for enclosing go.mod. + for { + if fi, err := fsys.Stat(filepath.Join(dir, "go.mod")); err == nil && !fi.IsDir() { + return dir + } + d := filepath.Dir(dir) + if d == dir { + break + } + dir = d + } + return "" +} + +func findWorkspaceFile(dir string) (root string) { + if dir == "" { + panic("dir not set") + } + dir = filepath.Clean(dir) + + // Look for enclosing go.mod. + for { + f := filepath.Join(dir, "go.work") + if fi, err := fsys.Stat(f); err == nil && !fi.IsDir() { + return f + } + d := filepath.Dir(dir) + if d == dir { + break + } + if d == cfg.GOROOT { + // As a special case, don't cross GOROOT to find a go.work file. + // The standard library and commands built in go always use the vendored + // dependencies, so avoid using a most likely irrelevant go.work file. + return "" + } + dir = d + } + return "" +} + +func findAltConfig(dir string) (root, name string) { + if dir == "" { + panic("dir not set") + } + dir = filepath.Clean(dir) + if rel := search.InDir(dir, cfg.BuildContext.GOROOT); rel != "" { + // Don't suggest creating a module from $GOROOT/.git/config + // or a config file found in any parent of $GOROOT (see #34191). + return "", "" + } + for { + for _, name := range altConfigs { + if fi, err := fsys.Stat(filepath.Join(dir, name)); err == nil && !fi.IsDir() { + return dir, name + } + } + d := filepath.Dir(dir) + if d == dir { + break + } + dir = d + } + return "", "" +} + +func findModulePath(dir string) (string, error) { + // TODO(bcmills): once we have located a plausible module path, we should + // query version control (if available) to verify that it matches the major + // version of the most recent tag. + // See https://golang.org/issue/29433, https://golang.org/issue/27009, and + // https://golang.org/issue/31549. + + // Cast about for import comments, + // first in top-level directory, then in subdirectories. + list, _ := os.ReadDir(dir) + for _, info := range list { + if info.Type().IsRegular() && strings.HasSuffix(info.Name(), ".go") { + if com := findImportComment(filepath.Join(dir, info.Name())); com != "" { + return com, nil + } + } + } + for _, info1 := range list { + if info1.IsDir() { + files, _ := os.ReadDir(filepath.Join(dir, info1.Name())) + for _, info2 := range files { + if info2.Type().IsRegular() && strings.HasSuffix(info2.Name(), ".go") { + if com := findImportComment(filepath.Join(dir, info1.Name(), info2.Name())); com != "" { + return path.Dir(com), nil + } + } + } + } + } + + // Look for Godeps.json declaring import path. + data, _ := os.ReadFile(filepath.Join(dir, "Godeps/Godeps.json")) + var cfg1 struct{ ImportPath string } + json.Unmarshal(data, &cfg1) + if cfg1.ImportPath != "" { + return cfg1.ImportPath, nil + } + + // Look for vendor.json declaring import path. + data, _ = os.ReadFile(filepath.Join(dir, "vendor/vendor.json")) + var cfg2 struct{ RootPath string } + json.Unmarshal(data, &cfg2) + if cfg2.RootPath != "" { + return cfg2.RootPath, nil + } + + // Look for path in GOPATH. + var badPathErr error + for _, gpdir := range filepath.SplitList(cfg.BuildContext.GOPATH) { + if gpdir == "" { + continue + } + if rel := search.InDir(dir, filepath.Join(gpdir, "src")); rel != "" && rel != "." { + path := filepath.ToSlash(rel) + // gorelease will alert users publishing their modules to fix their paths. + if err := module.CheckImportPath(path); err != nil { + badPathErr = err + break + } + return path, nil + } + } + + reason := "outside GOPATH, module path must be specified" + if badPathErr != nil { + // return a different error message if the module was in GOPATH, but + // the module path determined above would be an invalid path. + reason = fmt.Sprintf("bad module path inferred from directory in GOPATH: %v", badPathErr) + } + msg := `cannot determine module path for source directory %s (%s) + +Example usage: + 'go mod init example.com/m' to initialize a v0 or v1 module + 'go mod init example.com/m/v2' to initialize a v2 module + +Run 'go help mod init' for more information. +` + return "", fmt.Errorf(msg, dir, reason) +} + +var ( + importCommentRE = lazyregexp.New(`(?m)^package[ \t]+[^ \t\r\n/]+[ \t]+//[ \t]+import[ \t]+(\"[^"]+\")[ \t]*\r?\n`) +) + +func findImportComment(file string) string { + data, err := os.ReadFile(file) + if err != nil { + return "" + } + m := importCommentRE.FindSubmatch(data) + if m == nil { + return "" + } + path, err := strconv.Unquote(string(m[1])) + if err != nil { + return "" + } + return path +} + +// WriteOpts control the behavior of WriteGoMod. +type WriteOpts struct { + DropToolchain bool // go get toolchain@none + ExplicitToolchain bool // go get has set explicit toolchain version + + // TODO(bcmills): Make 'go mod tidy' update the go version in the Requirements + // instead of writing directly to the modfile.File + TidyWroteGo bool // Go.Version field already updated by 'go mod tidy' +} + +// WriteGoMod writes the current build list back to go.mod. +func WriteGoMod(ctx context.Context, opts WriteOpts) error { + requirements = LoadModFile(ctx) + return commitRequirements(ctx, opts) +} + +// commitRequirements ensures go.mod and go.sum are up to date with the current +// requirements. +// +// In "mod" mode, commitRequirements writes changes to go.mod and go.sum. +// +// In "readonly" and "vendor" modes, commitRequirements returns an error if +// go.mod or go.sum are out of date in a semantically significant way. +// +// In workspace mode, commitRequirements only writes changes to go.work.sum. +func commitRequirements(ctx context.Context, opts WriteOpts) (err error) { + if inWorkspaceMode() { + // go.mod files aren't updated in workspace mode, but we still want to + // update the go.work.sum file. + return modfetch.WriteGoSum(ctx, keepSums(ctx, loaded, requirements, addBuildListZipSums), mustHaveCompleteRequirements()) + } + if MainModules.Len() != 1 || MainModules.ModRoot(MainModules.Versions()[0]) == "" { + // We aren't in a module, so we don't have anywhere to write a go.mod file. + return nil + } + mainModule := MainModules.mustGetSingleMainModule() + modFile := MainModules.ModFile(mainModule) + if modFile == nil { + // command-line-arguments has no .mod file to write. + return nil + } + modFilePath := modFilePath(MainModules.ModRoot(mainModule)) + + var list []*modfile.Require + toolchain := "" + goVersion := "" + for _, m := range requirements.rootModules { + if m.Path == "go" { + goVersion = m.Version + continue + } + if m.Path == "toolchain" { + toolchain = m.Version + continue + } + list = append(list, &modfile.Require{ + Mod: m, + Indirect: !requirements.direct[m.Path], + }) + } + + // Update go line. + // Every MVS graph we consider should have go as a root, + // and toolchain is either implied by the go line or explicitly a root. + if goVersion == "" { + base.Fatalf("go: internal error: missing go root module in WriteGoMod") + } + if gover.Compare(goVersion, gover.Local()) > 0 { + // We cannot assume that we know how to update a go.mod to a newer version. + return &gover.TooNewError{What: "updating go.mod", GoVersion: goVersion} + } + wroteGo := opts.TidyWroteGo + if !wroteGo && modFile.Go == nil || modFile.Go.Version != goVersion { + alwaysUpdate := cfg.BuildMod == "mod" || cfg.CmdName == "mod tidy" || cfg.CmdName == "get" + if modFile.Go == nil && goVersion == gover.DefaultGoModVersion && !alwaysUpdate { + // The go.mod has no go line, the implied default Go version matches + // what we've computed for the graph, and we're not in one of the + // traditional go.mod-updating programs, so leave it alone. + } else { + wroteGo = true + forceGoStmt(modFile, mainModule, goVersion) + } + } + if toolchain == "" { + toolchain = "go" + goVersion + } + + // For reproducibility, if we are writing a new go line, + // and we're not explicitly modifying the toolchain line with 'go get toolchain@something', + // and the go version is one that supports switching toolchains, + // and the toolchain running right now is newer than the current toolchain line, + // then update the toolchain line to record the newer toolchain. + // + // TODO(#57001): This condition feels too complicated. Can we simplify it? + // TODO(#57001): Add more tests for toolchain lines. + toolVers := gover.FromToolchain(toolchain) + if wroteGo && !opts.DropToolchain && !opts.ExplicitToolchain && + gover.Compare(goVersion, gover.GoStrictVersion) >= 0 && + (gover.Compare(gover.Local(), toolVers) > 0 && !gover.IsLang(gover.Local())) { + toolchain = "go" + gover.Local() + toolVers = gover.FromToolchain(toolchain) + } + + if opts.DropToolchain || toolchain == "go"+goVersion || (gover.Compare(toolVers, gover.GoStrictVersion) < 0 && !opts.ExplicitToolchain) { + // go get toolchain@none or toolchain matches go line or isn't valid; drop it. + // TODO(#57001): 'go get' should reject explicit toolchains below GoStrictVersion. + modFile.DropToolchainStmt() + } else { + modFile.AddToolchainStmt(toolchain) + } + + // Update require blocks. + if gover.Compare(goVersion, gover.SeparateIndirectVersion) < 0 { + modFile.SetRequire(list) + } else { + modFile.SetRequireSeparateIndirect(list) + } + modFile.Cleanup() + + index := MainModules.GetSingleIndexOrNil() + dirty := index.modFileIsDirty(modFile) + if dirty && cfg.BuildMod != "mod" { + // If we're about to fail due to -mod=readonly, + // prefer to report a dirty go.mod over a dirty go.sum + return errGoModDirty + } + + if !dirty && cfg.CmdName != "mod tidy" { + // The go.mod file has the same semantic content that it had before + // (but not necessarily the same exact bytes). + // Don't write go.mod, but write go.sum in case we added or trimmed sums. + // 'go mod init' shouldn't write go.sum, since it will be incomplete. + if cfg.CmdName != "mod init" { + if err := modfetch.WriteGoSum(ctx, keepSums(ctx, loaded, requirements, addBuildListZipSums), mustHaveCompleteRequirements()); err != nil { + return err + } + } + return nil + } + if _, ok := fsys.OverlayPath(modFilePath); ok { + if dirty { + return errors.New("updates to go.mod needed, but go.mod is part of the overlay specified with -overlay") + } + return nil + } + + new, err := modFile.Format() + if err != nil { + return err + } + defer func() { + // At this point we have determined to make the go.mod file on disk equal to new. + MainModules.SetIndex(mainModule, indexModFile(new, modFile, mainModule, false)) + + // Update go.sum after releasing the side lock and refreshing the index. + // 'go mod init' shouldn't write go.sum, since it will be incomplete. + if cfg.CmdName != "mod init" { + if err == nil { + err = modfetch.WriteGoSum(ctx, keepSums(ctx, loaded, requirements, addBuildListZipSums), mustHaveCompleteRequirements()) + } + } + }() + + // Make a best-effort attempt to acquire the side lock, only to exclude + // previous versions of the 'go' command from making simultaneous edits. + if unlock, err := modfetch.SideLock(ctx); err == nil { + defer unlock() + } + + errNoChange := errors.New("no update needed") + + err = lockedfile.Transform(modFilePath, func(old []byte) ([]byte, error) { + if bytes.Equal(old, new) { + // The go.mod file is already equal to new, possibly as the result of some + // other process. + return nil, errNoChange + } + + if index != nil && !bytes.Equal(old, index.data) { + // The contents of the go.mod file have changed. In theory we could add all + // of the new modules to the build list, recompute, and check whether any + // module in *our* build list got bumped to a different version, but that's + // a lot of work for marginal benefit. Instead, fail the command: if users + // want to run concurrent commands, they need to start with a complete, + // consistent module definition. + return nil, fmt.Errorf("existing contents have changed since last read") + } + + return new, nil + }) + + if err != nil && err != errNoChange { + return fmt.Errorf("updating go.mod: %w", err) + } + return nil +} + +// keepSums returns the set of modules (and go.mod file entries) for which +// checksums would be needed in order to reload the same set of packages +// loaded by the most recent call to LoadPackages or ImportFromFiles, +// including any go.mod files needed to reconstruct the MVS result +// or identify go versions, +// in addition to the checksums for every module in keepMods. +func keepSums(ctx context.Context, ld *loader, rs *Requirements, which whichSums) map[module.Version]bool { + // Every module in the full module graph contributes its requirements, + // so in order to ensure that the build list itself is reproducible, + // we need sums for every go.mod in the graph (regardless of whether + // that version is selected). + keep := make(map[module.Version]bool) + + // Add entries for modules in the build list with paths that are prefixes of + // paths of loaded packages. We need to retain sums for all of these modules — + // not just the modules containing the actual packages — in order to rule out + // ambiguous import errors the next time we load the package. + keepModSumsForZipSums := true + if ld == nil { + if gover.Compare(MainModules.GoVersion(), gover.TidyGoModSumVersion) < 0 && cfg.BuildMod != "mod" { + keepModSumsForZipSums = false + } + } else { + keepPkgGoModSums := true + if gover.Compare(ld.requirements.GoVersion(), gover.TidyGoModSumVersion) < 0 && (ld.Tidy || cfg.BuildMod != "mod") { + keepPkgGoModSums = false + keepModSumsForZipSums = false + } + for _, pkg := range ld.pkgs { + // We check pkg.mod.Path here instead of pkg.inStd because the + // pseudo-package "C" is not in std, but not provided by any module (and + // shouldn't force loading the whole module graph). + if pkg.testOf != nil || (pkg.mod.Path == "" && pkg.err == nil) || module.CheckImportPath(pkg.path) != nil { + continue + } + + // We need the checksum for the go.mod file for pkg.mod + // so that we know what Go version to use to compile pkg. + // However, we didn't do so before Go 1.21, and the bug is relatively + // minor, so we maintain the previous (buggy) behavior in 'go mod tidy' to + // avoid introducing unnecessary churn. + if keepPkgGoModSums { + r := resolveReplacement(pkg.mod) + keep[modkey(r)] = true + } + + if rs.pruning == pruned && pkg.mod.Path != "" { + if v, ok := rs.rootSelected(pkg.mod.Path); ok && v == pkg.mod.Version { + // pkg was loaded from a root module, and because the main module has + // a pruned module graph we do not check non-root modules for + // conflicts for packages that can be found in roots. So we only need + // the checksums for the root modules that may contain pkg, not all + // possible modules. + for prefix := pkg.path; prefix != "."; prefix = path.Dir(prefix) { + if v, ok := rs.rootSelected(prefix); ok && v != "none" { + m := module.Version{Path: prefix, Version: v} + r := resolveReplacement(m) + keep[r] = true + } + } + continue + } + } + + mg, _ := rs.Graph(ctx) + for prefix := pkg.path; prefix != "."; prefix = path.Dir(prefix) { + if v := mg.Selected(prefix); v != "none" { + m := module.Version{Path: prefix, Version: v} + r := resolveReplacement(m) + keep[r] = true + } + } + } + } + + if rs.graph.Load() == nil { + // We haven't needed to load the module graph so far. + // Save sums for the root modules (or their replacements), but don't + // incur the cost of loading the graph just to find and retain the sums. + for _, m := range rs.rootModules { + r := resolveReplacement(m) + keep[modkey(r)] = true + if which == addBuildListZipSums { + keep[r] = true + } + } + } else { + mg, _ := rs.Graph(ctx) + mg.WalkBreadthFirst(func(m module.Version) { + if _, ok := mg.RequiredBy(m); ok { + // The requirements from m's go.mod file are present in the module graph, + // so they are relevant to the MVS result regardless of whether m was + // actually selected. + r := resolveReplacement(m) + keep[modkey(r)] = true + } + }) + + if which == addBuildListZipSums { + for _, m := range mg.BuildList() { + r := resolveReplacement(m) + if keepModSumsForZipSums { + keep[modkey(r)] = true // we need the go version from the go.mod file to do anything useful with the zipfile + } + keep[r] = true + } + } + } + + return keep +} + +type whichSums int8 + +const ( + loadedZipSumsOnly = whichSums(iota) + addBuildListZipSums +) + +// modkey returns the module.Version under which the checksum for m's go.mod +// file is stored in the go.sum file. +func modkey(m module.Version) module.Version { + return module.Version{Path: m.Path, Version: m.Version + "/go.mod"} +} + +func suggestModulePath(path string) string { + var m string + + i := len(path) + for i > 0 && ('0' <= path[i-1] && path[i-1] <= '9' || path[i-1] == '.') { + i-- + } + url := path[:i] + url = strings.TrimSuffix(url, "/v") + url = strings.TrimSuffix(url, "/") + + f := func(c rune) bool { + return c > '9' || c < '0' + } + s := strings.FieldsFunc(path[i:], f) + if len(s) > 0 { + m = s[0] + } + m = strings.TrimLeft(m, "0") + if m == "" || m == "1" { + return url + "/v2" + } + + return url + "/v" + m +} + +func suggestGopkgIn(path string) string { + var m string + i := len(path) + for i > 0 && (('0' <= path[i-1] && path[i-1] <= '9') || (path[i-1] == '.')) { + i-- + } + url := path[:i] + url = strings.TrimSuffix(url, ".v") + url = strings.TrimSuffix(url, "/v") + url = strings.TrimSuffix(url, "/") + + f := func(c rune) bool { + return c > '9' || c < '0' + } + s := strings.FieldsFunc(path, f) + if len(s) > 0 { + m = s[0] + } + + m = strings.TrimLeft(m, "0") + + if m == "" { + return url + ".v1" + } + return url + ".v" + m +} diff --git a/src/cmd/go/internal/modload/list.go b/src/cmd/go/internal/modload/list.go new file mode 100644 index 0000000..ef93c25 --- /dev/null +++ b/src/cmd/go/internal/modload/list.go @@ -0,0 +1,310 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "runtime" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/gover" + "cmd/go/internal/modfetch/codehost" + "cmd/go/internal/modinfo" + "cmd/go/internal/search" + "cmd/internal/pkgpattern" + + "golang.org/x/mod/module" +) + +type ListMode int + +const ( + ListU ListMode = 1 << iota + ListRetracted + ListDeprecated + ListVersions + ListRetractedVersions +) + +// ListModules returns a description of the modules matching args, if known, +// along with any error preventing additional matches from being identified. +// +// The returned slice can be nonempty even if the error is non-nil. +func ListModules(ctx context.Context, args []string, mode ListMode, reuseFile string) ([]*modinfo.ModulePublic, error) { + var reuse map[module.Version]*modinfo.ModulePublic + if reuseFile != "" { + data, err := os.ReadFile(reuseFile) + if err != nil { + return nil, err + } + dec := json.NewDecoder(bytes.NewReader(data)) + reuse = make(map[module.Version]*modinfo.ModulePublic) + for { + var m modinfo.ModulePublic + if err := dec.Decode(&m); err != nil { + if err == io.EOF { + break + } + return nil, fmt.Errorf("parsing %s: %v", reuseFile, err) + } + if m.Origin == nil { + continue + } + m.Reuse = true + reuse[module.Version{Path: m.Path, Version: m.Version}] = &m + if m.Query != "" { + reuse[module.Version{Path: m.Path, Version: m.Query}] = &m + } + } + } + + rs, mods, err := listModules(ctx, LoadModFile(ctx), args, mode, reuse) + + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + if mode != 0 { + for _, m := range mods { + if m.Reuse { + continue + } + add := func(m *modinfo.ModulePublic) { + sem <- token{} + go func() { + if mode&ListU != 0 { + addUpdate(ctx, m) + } + if mode&ListVersions != 0 { + addVersions(ctx, m, mode&ListRetractedVersions != 0) + } + if mode&ListRetracted != 0 { + addRetraction(ctx, m) + } + if mode&ListDeprecated != 0 { + addDeprecation(ctx, m) + } + <-sem + }() + } + + add(m) + if m.Replace != nil { + add(m.Replace) + } + } + } + // Fill semaphore channel to wait for all tasks to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} + } + + if err == nil { + requirements = rs + // TODO(#61605): The extra ListU clause fixes a problem with Go 1.21rc3 + // where "go mod tidy" and "go list -m -u all" fight over whether the go.sum + // should be considered up-to-date. The fix for now is to always treat the + // go.sum as up-to-date during list -m -u. Probably the right fix is more targeted, + // but in general list -u is looking up other checksums in the checksum database + // that won't be necessary later, so it makes sense not to write the go.sum back out. + if !ExplicitWriteGoMod && mode&ListU == 0 { + err = commitRequirements(ctx, WriteOpts{}) + } + } + return mods, err +} + +func listModules(ctx context.Context, rs *Requirements, args []string, mode ListMode, reuse map[module.Version]*modinfo.ModulePublic) (_ *Requirements, mods []*modinfo.ModulePublic, mgErr error) { + if len(args) == 0 { + var ms []*modinfo.ModulePublic + for _, m := range MainModules.Versions() { + if gover.IsToolchain(m.Path) { + continue + } + ms = append(ms, moduleInfo(ctx, rs, m, mode, reuse)) + } + return rs, ms, nil + } + + needFullGraph := false + for _, arg := range args { + if strings.Contains(arg, `\`) { + base.Fatalf("go: module paths never use backslash") + } + if search.IsRelativePath(arg) { + base.Fatalf("go: cannot use relative path %s to specify module", arg) + } + if arg == "all" || strings.Contains(arg, "...") { + needFullGraph = true + if !HasModRoot() { + base.Fatalf("go: cannot match %q: %v", arg, ErrNoModRoot) + } + continue + } + if path, vers, found := strings.Cut(arg, "@"); found { + if vers == "upgrade" || vers == "patch" { + if _, ok := rs.rootSelected(path); !ok || rs.pruning == unpruned { + needFullGraph = true + if !HasModRoot() { + base.Fatalf("go: cannot match %q: %v", arg, ErrNoModRoot) + } + } + } + continue + } + if _, ok := rs.rootSelected(arg); !ok || rs.pruning == unpruned { + needFullGraph = true + if mode&ListVersions == 0 && !HasModRoot() { + base.Fatalf("go: cannot match %q without -versions or an explicit version: %v", arg, ErrNoModRoot) + } + } + } + + var mg *ModuleGraph + if needFullGraph { + rs, mg, mgErr = expandGraph(ctx, rs) + } + + matchedModule := map[module.Version]bool{} + for _, arg := range args { + if path, vers, found := strings.Cut(arg, "@"); found { + var current string + if mg == nil { + current, _ = rs.rootSelected(path) + } else { + current = mg.Selected(path) + } + if current == "none" && mgErr != nil { + if vers == "upgrade" || vers == "patch" { + // The module graph is incomplete, so we don't know what version we're + // actually upgrading from. + // mgErr is already set, so just skip this module. + continue + } + } + + allowed := CheckAllowed + if IsRevisionQuery(path, vers) || mode&ListRetracted != 0 { + // Allow excluded and retracted versions if the user asked for a + // specific revision or used 'go list -retracted'. + allowed = nil + } + info, err := queryReuse(ctx, path, vers, current, allowed, reuse) + if err != nil { + var origin *codehost.Origin + if info != nil { + origin = info.Origin + } + mods = append(mods, &modinfo.ModulePublic{ + Path: path, + Version: vers, + Error: modinfoError(path, vers, err), + Origin: origin, + }) + continue + } + + // Indicate that m was resolved from outside of rs by passing a nil + // *Requirements instead. + var noRS *Requirements + + mod := moduleInfo(ctx, noRS, module.Version{Path: path, Version: info.Version}, mode, reuse) + if vers != mod.Version { + mod.Query = vers + } + mod.Origin = info.Origin + mods = append(mods, mod) + continue + } + + // Module path or pattern. + var match func(string) bool + if arg == "all" { + match = func(p string) bool { return !gover.IsToolchain(p) } + } else if strings.Contains(arg, "...") { + mp := pkgpattern.MatchPattern(arg) + match = func(p string) bool { return mp(p) && !gover.IsToolchain(p) } + } else { + var v string + if mg == nil { + var ok bool + v, ok = rs.rootSelected(arg) + if !ok { + // We checked rootSelected(arg) in the earlier args loop, so if there + // is no such root we should have loaded a non-nil mg. + panic(fmt.Sprintf("internal error: root requirement expected but not found for %v", arg)) + } + } else { + v = mg.Selected(arg) + } + if v == "none" && mgErr != nil { + // mgErr is already set, so just skip this module. + continue + } + if v != "none" { + mods = append(mods, moduleInfo(ctx, rs, module.Version{Path: arg, Version: v}, mode, reuse)) + } else if cfg.BuildMod == "vendor" { + // In vendor mode, we can't determine whether a missing module is “a + // known dependency” because the module graph is incomplete. + // Give a more explicit error message. + mods = append(mods, &modinfo.ModulePublic{ + Path: arg, + Error: modinfoError(arg, "", errors.New("can't resolve module using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)")), + }) + } else if mode&ListVersions != 0 { + // Don't make the user provide an explicit '@latest' when they're + // explicitly asking what the available versions are. Instead, return a + // module with version "none", to which we can add the requested list. + mods = append(mods, &modinfo.ModulePublic{Path: arg}) + } else { + mods = append(mods, &modinfo.ModulePublic{ + Path: arg, + Error: modinfoError(arg, "", errors.New("not a known dependency")), + }) + } + continue + } + + matched := false + for _, m := range mg.BuildList() { + if match(m.Path) { + matched = true + if !matchedModule[m] { + matchedModule[m] = true + mods = append(mods, moduleInfo(ctx, rs, m, mode, reuse)) + } + } + } + if !matched { + fmt.Fprintf(os.Stderr, "warning: pattern %q matched no module dependencies\n", arg) + } + } + + return rs, mods, mgErr +} + +// modinfoError wraps an error to create an error message in +// modinfo.ModuleError with minimal redundancy. +func modinfoError(path, vers string, err error) *modinfo.ModuleError { + var nerr *NoMatchingVersionError + var merr *module.ModuleError + if errors.As(err, &nerr) { + // NoMatchingVersionError contains the query, so we don't mention the + // query again in ModuleError. + err = &module.ModuleError{Path: path, Err: err} + } else if !errors.As(err, &merr) { + // If the error does not contain path and version, wrap it in a + // module.ModuleError. + err = &module.ModuleError{Path: path, Version: vers, Err: err} + } + + return &modinfo.ModuleError{Err: err.Error()} +} diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go new file mode 100644 index 0000000..51eb141 --- /dev/null +++ b/src/cmd/go/internal/modload/load.go @@ -0,0 +1,2352 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +// This file contains the module-mode package loader, as well as some accessory +// functions pertaining to the package import graph. +// +// There are two exported entry points into package loading — LoadPackages and +// ImportFromFiles — both implemented in terms of loadFromRoots, which itself +// manipulates an instance of the loader struct. +// +// Although most of the loading state is maintained in the loader struct, +// one key piece - the build list - is a global, so that it can be modified +// separate from the loading operation, such as during "go get" +// upgrades/downgrades or in "go mod" operations. +// TODO(#40775): It might be nice to make the loader take and return +// a buildList rather than hard-coding use of the global. +// +// Loading is an iterative process. On each iteration, we try to load the +// requested packages and their transitive imports, then try to resolve modules +// for any imported packages that are still missing. +// +// The first step of each iteration identifies a set of “root” packages. +// Normally the root packages are exactly those matching the named pattern +// arguments. However, for the "all" meta-pattern, the final set of packages is +// computed from the package import graph, and therefore cannot be an initial +// input to loading that graph. Instead, the root packages for the "all" pattern +// are those contained in the main module, and allPatternIsRoot parameter to the +// loader instructs it to dynamically expand those roots to the full "all" +// pattern as loading progresses. +// +// The pkgInAll flag on each loadPkg instance tracks whether that +// package is known to match the "all" meta-pattern. +// A package matches the "all" pattern if: +// - it is in the main module, or +// - it is imported by any test in the main module, or +// - it is imported by another package in "all", or +// - the main module specifies a go version ≤ 1.15, and the package is imported +// by a *test of* another package in "all". +// +// When graph pruning is in effect, we want to spot-check the graph-pruning +// invariants — which depend on which packages are known to be in "all" — even +// when we are only loading individual packages, so we set the pkgInAll flag +// regardless of the whether the "all" pattern is a root. +// (This is necessary to maintain the “import invariant” described in +// https://golang.org/design/36460-lazy-module-loading.) +// +// Because "go mod vendor" prunes out the tests of vendored packages, the +// behavior of the "all" pattern with -mod=vendor in Go 1.11–1.15 is the same +// as the "all" pattern (regardless of the -mod flag) in 1.16+. +// The loader uses the GoVersion parameter to determine whether the "all" +// pattern should close over tests (as in Go 1.11–1.15) or stop at only those +// packages transitively imported by the packages and tests in the main module +// ("all" in Go 1.16+ and "go mod vendor" in Go 1.11+). +// +// Note that it is possible for a loaded package NOT to be in "all" even when we +// are loading the "all" pattern. For example, packages that are transitive +// dependencies of other roots named on the command line must be loaded, but are +// not in "all". (The mod_notall test illustrates this behavior.) +// Similarly, if the LoadTests flag is set but the "all" pattern does not close +// over test dependencies, then when we load the test of a package that is in +// "all" but outside the main module, the dependencies of that test will not +// necessarily themselves be in "all". (That configuration does not arise in Go +// 1.11–1.15, but it will be possible in Go 1.16+.) +// +// Loading proceeds from the roots, using a parallel work-queue with a limit on +// the amount of active work (to avoid saturating disks, CPU cores, and/or +// network connections). Each package is added to the queue the first time it is +// imported by another package. When we have finished identifying the imports of +// a package, we add the test for that package if it is needed. A test may be +// needed if: +// - the package matches a root pattern and tests of the roots were requested, or +// - the package is in the main module and the "all" pattern is requested +// (because the "all" pattern includes the dependencies of tests in the main +// module), or +// - the package is in "all" and the definition of "all" we are using includes +// dependencies of tests (as is the case in Go ≤1.15). +// +// After all available packages have been loaded, we examine the results to +// identify any requested or imported packages that are still missing, and if +// so, which modules we could add to the module graph in order to make the +// missing packages available. We add those to the module graph and iterate, +// until either all packages resolve successfully or we cannot identify any +// module that would resolve any remaining missing package. +// +// If the main module is “tidy” (that is, if "go mod tidy" is a no-op for it) +// and all requested packages are in "all", then loading completes in a single +// iteration. +// TODO(bcmills): We should also be able to load in a single iteration if the +// requested packages all come from modules that are themselves tidy, regardless +// of whether those packages are in "all". Today, that requires two iterations +// if those packages are not found in existing dependencies of the main module. + +import ( + "context" + "errors" + "fmt" + "go/build" + "io/fs" + "os" + "path" + pathpkg "path" + "path/filepath" + "reflect" + "runtime" + "sort" + "strings" + "sync" + "sync/atomic" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/gover" + "cmd/go/internal/imports" + "cmd/go/internal/modfetch" + "cmd/go/internal/modindex" + "cmd/go/internal/mvs" + "cmd/go/internal/par" + "cmd/go/internal/search" + "cmd/go/internal/str" + + "golang.org/x/mod/module" +) + +// loaded is the most recently-used package loader. +// It holds details about individual packages. +// +// This variable should only be accessed directly in top-level exported +// functions. All other functions that require or produce a *loader should pass +// or return it as an explicit parameter. +var loaded *loader + +// PackageOpts control the behavior of the LoadPackages function. +type PackageOpts struct { + // TidyGoVersion is the Go version to which the go.mod file should be updated + // after packages have been loaded. + // + // An empty TidyGoVersion means to use the Go version already specified in the + // main module's go.mod file, or the latest Go version if there is no main + // module. + TidyGoVersion string + + // Tags are the build tags in effect (as interpreted by the + // cmd/go/internal/imports package). + // If nil, treated as equivalent to imports.Tags(). + Tags map[string]bool + + // Tidy, if true, requests that the build list and go.sum file be reduced to + // the minimal dependencies needed to reproducibly reload the requested + // packages. + Tidy bool + + // TidyCompatibleVersion is the oldest Go version that must be able to + // reproducibly reload the requested packages. + // + // If empty, the compatible version is the Go version immediately prior to the + // 'go' version listed in the go.mod file. + TidyCompatibleVersion string + + // VendorModulesInGOROOTSrc indicates that if we are within a module in + // GOROOT/src, packages in the module's vendor directory should be resolved as + // actual module dependencies (instead of standard-library packages). + VendorModulesInGOROOTSrc bool + + // ResolveMissingImports indicates that we should attempt to add module + // dependencies as needed to resolve imports of packages that are not found. + // + // For commands that support the -mod flag, resolving imports may still fail + // if the flag is set to "readonly" (the default) or "vendor". + ResolveMissingImports bool + + // AssumeRootsImported indicates that the transitive dependencies of the root + // packages should be treated as if those roots will be imported by the main + // module. + AssumeRootsImported bool + + // AllowPackage, if non-nil, is called after identifying the module providing + // each package. If AllowPackage returns a non-nil error, that error is set + // for the package, and the imports and test of that package will not be + // loaded. + // + // AllowPackage may be invoked concurrently by multiple goroutines, + // and may be invoked multiple times for a given package path. + AllowPackage func(ctx context.Context, path string, mod module.Version) error + + // LoadTests loads the test dependencies of each package matching a requested + // pattern. If ResolveMissingImports is also true, test dependencies will be + // resolved if missing. + LoadTests bool + + // UseVendorAll causes the "all" package pattern to be interpreted as if + // running "go mod vendor" (or building with "-mod=vendor"). + // + // This is a no-op for modules that declare 'go 1.16' or higher, for which this + // is the default (and only) interpretation of the "all" pattern in module mode. + UseVendorAll bool + + // AllowErrors indicates that LoadPackages should not terminate the process if + // an error occurs. + AllowErrors bool + + // SilencePackageErrors indicates that LoadPackages should not print errors + // that occur while matching or loading packages, and should not terminate the + // process if such an error occurs. + // + // Errors encountered in the module graph will still be reported. + // + // The caller may retrieve the silenced package errors using the Lookup + // function, and matching errors are still populated in the Errs field of the + // associated search.Match.) + SilencePackageErrors bool + + // SilenceMissingStdImports indicates that LoadPackages should not print + // errors or terminate the process if an imported package is missing, and the + // import path looks like it might be in the standard library (perhaps in a + // future version). + SilenceMissingStdImports bool + + // SilenceNoGoErrors indicates that LoadPackages should not print + // imports.ErrNoGo errors. + // This allows the caller to invoke LoadPackages (and report other errors) + // without knowing whether the requested packages exist for the given tags. + // + // Note that if a requested package does not exist *at all*, it will fail + // during module resolution and the error will not be suppressed. + SilenceNoGoErrors bool + + // SilenceUnmatchedWarnings suppresses the warnings normally emitted for + // patterns that did not match any packages. + SilenceUnmatchedWarnings bool + + // Resolve the query against this module. + MainModule module.Version + + // If Switcher is non-nil, then LoadPackages passes all encountered errors + // to Switcher.Error and tries Switcher.Switch before base.ExitIfErrors. + Switcher gover.Switcher +} + +// LoadPackages identifies the set of packages matching the given patterns and +// loads the packages in the import graph rooted at that set. +func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (matches []*search.Match, loadedPackages []string) { + if opts.Tags == nil { + opts.Tags = imports.Tags() + } + + patterns = search.CleanPatterns(patterns) + matches = make([]*search.Match, 0, len(patterns)) + allPatternIsRoot := false + for _, pattern := range patterns { + matches = append(matches, search.NewMatch(pattern)) + if pattern == "all" { + allPatternIsRoot = true + } + } + + updateMatches := func(rs *Requirements, ld *loader) { + for _, m := range matches { + switch { + case m.IsLocal(): + // Evaluate list of file system directories on first iteration. + if m.Dirs == nil { + matchModRoots := modRoots + if opts.MainModule != (module.Version{}) { + matchModRoots = []string{MainModules.ModRoot(opts.MainModule)} + } + matchLocalDirs(ctx, matchModRoots, m, rs) + } + + // Make a copy of the directory list and translate to import paths. + // Note that whether a directory corresponds to an import path + // changes as the build list is updated, and a directory can change + // from not being in the build list to being in it and back as + // the exact version of a particular module increases during + // the loader iterations. + m.Pkgs = m.Pkgs[:0] + for _, dir := range m.Dirs { + pkg, err := resolveLocalPackage(ctx, dir, rs) + if err != nil { + if !m.IsLiteral() && (err == errPkgIsBuiltin || err == errPkgIsGorootSrc) { + continue // Don't include "builtin" or GOROOT/src in wildcard patterns. + } + + // If we're outside of a module, ensure that the failure mode + // indicates that. + if !HasModRoot() { + die() + } + + if ld != nil { + m.AddError(err) + } + continue + } + m.Pkgs = append(m.Pkgs, pkg) + } + + case m.IsLiteral(): + m.Pkgs = []string{m.Pattern()} + + case strings.Contains(m.Pattern(), "..."): + m.Errs = m.Errs[:0] + mg, err := rs.Graph(ctx) + if err != nil { + // The module graph is (or may be) incomplete — perhaps we failed to + // load the requirements of some module. This is an error in matching + // the patterns to packages, because we may be missing some packages + // or we may erroneously match packages in the wrong versions of + // modules. However, for cases like 'go list -e', the error should not + // necessarily prevent us from loading the packages we could find. + m.Errs = append(m.Errs, err) + } + matchPackages(ctx, m, opts.Tags, includeStd, mg.BuildList()) + + case m.Pattern() == "all": + if ld == nil { + // The initial roots are the packages in the main module. + // loadFromRoots will expand that to "all". + m.Errs = m.Errs[:0] + matchModules := MainModules.Versions() + if opts.MainModule != (module.Version{}) { + matchModules = []module.Version{opts.MainModule} + } + matchPackages(ctx, m, opts.Tags, omitStd, matchModules) + } else { + // Starting with the packages in the main module, + // enumerate the full list of "all". + m.Pkgs = ld.computePatternAll() + } + + case m.Pattern() == "std" || m.Pattern() == "cmd": + if m.Pkgs == nil { + m.MatchPackages() // Locate the packages within GOROOT/src. + } + + default: + panic(fmt.Sprintf("internal error: modload missing case for pattern %s", m.Pattern())) + } + } + } + + initialRS, err := loadModFile(ctx, &opts) + if err != nil { + base.Fatal(err) + } + + ld := loadFromRoots(ctx, loaderParams{ + PackageOpts: opts, + requirements: initialRS, + + allPatternIsRoot: allPatternIsRoot, + + listRoots: func(rs *Requirements) (roots []string) { + updateMatches(rs, nil) + for _, m := range matches { + roots = append(roots, m.Pkgs...) + } + return roots + }, + }) + + // One last pass to finalize wildcards. + updateMatches(ld.requirements, ld) + + // List errors in matching patterns (such as directory permission + // errors for wildcard patterns). + if !ld.SilencePackageErrors { + for _, match := range matches { + for _, err := range match.Errs { + ld.error(err) + } + } + } + ld.exitIfErrors(ctx) + + if !opts.SilenceUnmatchedWarnings { + search.WarnUnmatched(matches) + } + + if opts.Tidy { + if cfg.BuildV { + mg, _ := ld.requirements.Graph(ctx) + for _, m := range initialRS.rootModules { + var unused bool + if ld.requirements.pruning == unpruned { + // m is unused if it was dropped from the module graph entirely. If it + // was only demoted from direct to indirect, it may still be in use via + // a transitive import. + unused = mg.Selected(m.Path) == "none" + } else { + // m is unused if it was dropped from the roots. If it is still present + // as a transitive dependency, that transitive dependency is not needed + // by any package or test in the main module. + _, ok := ld.requirements.rootSelected(m.Path) + unused = !ok + } + if unused { + fmt.Fprintf(os.Stderr, "unused %s\n", m.Path) + } + } + } + + keep := keepSums(ctx, ld, ld.requirements, loadedZipSumsOnly) + compatVersion := ld.TidyCompatibleVersion + goVersion := ld.requirements.GoVersion() + if compatVersion == "" { + if gover.Compare(goVersion, gover.GoStrictVersion) < 0 { + compatVersion = gover.Prev(goVersion) + } else { + // Starting at GoStrictVersion, we no longer maintain compatibility with + // versions older than what is listed in the go.mod file. + compatVersion = goVersion + } + } + if gover.Compare(compatVersion, goVersion) > 0 { + // Each version of the Go toolchain knows how to interpret go.mod and + // go.sum files produced by all previous versions, so a compatibility + // version higher than the go.mod version adds nothing. + compatVersion = goVersion + } + if compatPruning := pruningForGoVersion(compatVersion); compatPruning != ld.requirements.pruning { + compatRS := newRequirements(compatPruning, ld.requirements.rootModules, ld.requirements.direct) + ld.checkTidyCompatibility(ctx, compatRS, compatVersion) + + for m := range keepSums(ctx, ld, compatRS, loadedZipSumsOnly) { + keep[m] = true + } + } + + if !ExplicitWriteGoMod { + modfetch.TrimGoSum(keep) + + // commitRequirements below will also call WriteGoSum, but the "keep" map + // we have here could be strictly larger: commitRequirements only commits + // loaded.requirements, but here we may have also loaded (and want to + // preserve checksums for) additional entities from compatRS, which are + // only needed for compatibility with ld.TidyCompatibleVersion. + if err := modfetch.WriteGoSum(ctx, keep, mustHaveCompleteRequirements()); err != nil { + base.Fatal(err) + } + } + } + + // Success! Update go.mod and go.sum (if needed) and return the results. + // We'll skip updating if ExplicitWriteGoMod is true (the caller has opted + // to call WriteGoMod itself) or if ResolveMissingImports is false (the + // command wants to examine the package graph as-is). + loaded = ld + requirements = loaded.requirements + + for _, pkg := range ld.pkgs { + if !pkg.isTest() { + loadedPackages = append(loadedPackages, pkg.path) + } + } + sort.Strings(loadedPackages) + + if !ExplicitWriteGoMod && opts.ResolveMissingImports { + if err := commitRequirements(ctx, WriteOpts{}); err != nil { + base.Fatal(err) + } + } + + return matches, loadedPackages +} + +// matchLocalDirs is like m.MatchDirs, but tries to avoid scanning directories +// outside of the standard library and active modules. +func matchLocalDirs(ctx context.Context, modRoots []string, m *search.Match, rs *Requirements) { + if !m.IsLocal() { + panic(fmt.Sprintf("internal error: resolveLocalDirs on non-local pattern %s", m.Pattern())) + } + + if i := strings.Index(m.Pattern(), "..."); i >= 0 { + // The pattern is local, but it is a wildcard. Its packages will + // only resolve to paths if they are inside of the standard + // library, the main module, or some dependency of the main + // module. Verify that before we walk the filesystem: a filesystem + // walk in a directory like /var or /etc can be very expensive! + dir := filepath.Dir(filepath.Clean(m.Pattern()[:i+3])) + absDir := dir + if !filepath.IsAbs(dir) { + absDir = filepath.Join(base.Cwd(), dir) + } + + modRoot := findModuleRoot(absDir) + found := false + for _, mainModuleRoot := range modRoots { + if mainModuleRoot == modRoot { + found = true + break + } + } + if !found && search.InDir(absDir, cfg.GOROOTsrc) == "" && pathInModuleCache(ctx, absDir, rs) == "" { + m.Dirs = []string{} + scope := "main module or its selected dependencies" + if inWorkspaceMode() { + scope = "modules listed in go.work or their selected dependencies" + } + m.AddError(fmt.Errorf("directory prefix %s does not contain %s", base.ShortPath(absDir), scope)) + return + } + } + + m.MatchDirs(modRoots) +} + +// resolveLocalPackage resolves a filesystem path to a package path. +func resolveLocalPackage(ctx context.Context, dir string, rs *Requirements) (string, error) { + var absDir string + if filepath.IsAbs(dir) { + absDir = filepath.Clean(dir) + } else { + absDir = filepath.Join(base.Cwd(), dir) + } + + bp, err := cfg.BuildContext.ImportDir(absDir, 0) + if err != nil && (bp == nil || len(bp.IgnoredGoFiles) == 0) { + // golang.org/issue/32917: We should resolve a relative path to a + // package path only if the relative path actually contains the code + // for that package. + // + // If the named directory does not exist or contains no Go files, + // the package does not exist. + // Other errors may affect package loading, but not resolution. + if _, err := fsys.Stat(absDir); err != nil { + if os.IsNotExist(err) { + // Canonicalize OS-specific errors to errDirectoryNotFound so that error + // messages will be easier for users to search for. + return "", &fs.PathError{Op: "stat", Path: absDir, Err: errDirectoryNotFound} + } + return "", err + } + if _, noGo := err.(*build.NoGoError); noGo { + // A directory that does not contain any Go source files — even ignored + // ones! — is not a Go package, and we can't resolve it to a package + // path because that path could plausibly be provided by some other + // module. + // + // Any other error indicates that the package “exists” (at least in the + // sense that it cannot exist in any other module), but has some other + // problem (such as a syntax error). + return "", err + } + } + + for _, mod := range MainModules.Versions() { + modRoot := MainModules.ModRoot(mod) + if modRoot != "" && absDir == modRoot { + if absDir == cfg.GOROOTsrc { + return "", errPkgIsGorootSrc + } + return MainModules.PathPrefix(mod), nil + } + } + + // Note: The checks for @ here are just to avoid misinterpreting + // the module cache directories (formerly GOPATH/src/mod/foo@v1.5.2/bar). + // It's not strictly necessary but helpful to keep the checks. + var pkgNotFoundErr error + pkgNotFoundLongestPrefix := "" + for _, mainModule := range MainModules.Versions() { + modRoot := MainModules.ModRoot(mainModule) + if modRoot != "" && str.HasFilePathPrefix(absDir, modRoot) && !strings.Contains(absDir[len(modRoot):], "@") { + suffix := filepath.ToSlash(str.TrimFilePathPrefix(absDir, modRoot)) + if pkg, found := strings.CutPrefix(suffix, "vendor/"); found { + if cfg.BuildMod != "vendor" { + return "", fmt.Errorf("without -mod=vendor, directory %s has no package path", absDir) + } + + readVendorList(VendorDir()) + if _, ok := vendorPkgModule[pkg]; !ok { + return "", fmt.Errorf("directory %s is not a package listed in vendor/modules.txt", absDir) + } + return pkg, nil + } + + mainModulePrefix := MainModules.PathPrefix(mainModule) + if mainModulePrefix == "" { + pkg := suffix + if pkg == "builtin" { + // "builtin" is a pseudo-package with a real source file. + // It's not included in "std", so it shouldn't resolve from "." + // within module "std" either. + return "", errPkgIsBuiltin + } + return pkg, nil + } + + pkg := pathpkg.Join(mainModulePrefix, suffix) + if _, ok, err := dirInModule(pkg, mainModulePrefix, modRoot, true); err != nil { + return "", err + } else if !ok { + // This main module could contain the directory but doesn't. Other main + // modules might contain the directory, so wait till we finish the loop + // to see if another main module contains directory. But if not, + // return an error. + if len(mainModulePrefix) > len(pkgNotFoundLongestPrefix) { + pkgNotFoundLongestPrefix = mainModulePrefix + pkgNotFoundErr = &PackageNotInModuleError{MainModules: []module.Version{mainModule}, Pattern: pkg} + } + continue + } + return pkg, nil + } + } + if pkgNotFoundErr != nil { + return "", pkgNotFoundErr + } + + if sub := search.InDir(absDir, cfg.GOROOTsrc); sub != "" && sub != "." && !strings.Contains(sub, "@") { + pkg := filepath.ToSlash(sub) + if pkg == "builtin" { + return "", errPkgIsBuiltin + } + return pkg, nil + } + + pkg := pathInModuleCache(ctx, absDir, rs) + if pkg == "" { + dirstr := fmt.Sprintf("directory %s", base.ShortPath(absDir)) + if dirstr == "directory ." { + dirstr = "current directory" + } + if inWorkspaceMode() { + if mr := findModuleRoot(absDir); mr != "" { + return "", fmt.Errorf("%s is contained in a module that is not one of the workspace modules listed in go.work. You can add the module to the workspace using:\n\tgo work use %s", dirstr, base.ShortPath(mr)) + } + return "", fmt.Errorf("%s outside modules listed in go.work or their selected dependencies", dirstr) + } + return "", fmt.Errorf("%s outside main module or its selected dependencies", dirstr) + } + return pkg, nil +} + +var ( + errDirectoryNotFound = errors.New("directory not found") + errPkgIsGorootSrc = errors.New("GOROOT/src is not an importable package") + errPkgIsBuiltin = errors.New(`"builtin" is a pseudo-package, not an importable package`) +) + +// pathInModuleCache returns the import path of the directory dir, +// if dir is in the module cache copy of a module in our build list. +func pathInModuleCache(ctx context.Context, dir string, rs *Requirements) string { + tryMod := func(m module.Version) (string, bool) { + if gover.IsToolchain(m.Path) { + return "", false + } + var root string + var err error + if repl := Replacement(m); repl.Path != "" && repl.Version == "" { + root = repl.Path + if !filepath.IsAbs(root) { + root = filepath.Join(replaceRelativeTo(), root) + } + } else if repl.Path != "" { + root, err = modfetch.DownloadDir(ctx, repl) + } else { + root, err = modfetch.DownloadDir(ctx, m) + } + if err != nil { + return "", false + } + + sub := search.InDir(dir, root) + if sub == "" { + return "", false + } + sub = filepath.ToSlash(sub) + if strings.Contains(sub, "/vendor/") || strings.HasPrefix(sub, "vendor/") || strings.Contains(sub, "@") { + return "", false + } + + return path.Join(m.Path, filepath.ToSlash(sub)), true + } + + if rs.pruning == pruned { + for _, m := range rs.rootModules { + if v, _ := rs.rootSelected(m.Path); v != m.Version { + continue // m is a root, but we have a higher root for the same path. + } + if importPath, ok := tryMod(m); ok { + // checkMultiplePaths ensures that a module can be used for at most one + // requirement, so this must be it. + return importPath + } + } + } + + // None of the roots contained dir, or the graph is unpruned (so we don't want + // to distinguish between roots and transitive dependencies). Either way, + // check the full graph to see if the directory is a non-root dependency. + // + // If the roots are not consistent with the full module graph, the selected + // versions of root modules may differ from what we already checked above. + // Re-check those paths too. + + mg, _ := rs.Graph(ctx) + var importPath string + for _, m := range mg.BuildList() { + var found bool + importPath, found = tryMod(m) + if found { + break + } + } + return importPath +} + +// ImportFromFiles adds modules to the build list as needed +// to satisfy the imports in the named Go source files. +// +// Errors in missing dependencies are silenced. +// +// TODO(bcmills): Silencing errors seems off. Take a closer look at this and +// figure out what the error-reporting actually ought to be. +func ImportFromFiles(ctx context.Context, gofiles []string) { + rs := LoadModFile(ctx) + + tags := imports.Tags() + imports, testImports, err := imports.ScanFiles(gofiles, tags) + if err != nil { + base.Fatal(err) + } + + loaded = loadFromRoots(ctx, loaderParams{ + PackageOpts: PackageOpts{ + Tags: tags, + ResolveMissingImports: true, + SilencePackageErrors: true, + }, + requirements: rs, + listRoots: func(*Requirements) (roots []string) { + roots = append(roots, imports...) + roots = append(roots, testImports...) + return roots + }, + }) + requirements = loaded.requirements + + if !ExplicitWriteGoMod { + if err := commitRequirements(ctx, WriteOpts{}); err != nil { + base.Fatal(err) + } + } +} + +// DirImportPath returns the effective import path for dir, +// provided it is within a main module, or else returns ".". +func (mms *MainModuleSet) DirImportPath(ctx context.Context, dir string) (path string, m module.Version) { + if !HasModRoot() { + return ".", module.Version{} + } + LoadModFile(ctx) // Sets targetPrefix. + + if !filepath.IsAbs(dir) { + dir = filepath.Join(base.Cwd(), dir) + } else { + dir = filepath.Clean(dir) + } + + var longestPrefix string + var longestPrefixPath string + var longestPrefixVersion module.Version + for _, v := range mms.Versions() { + modRoot := mms.ModRoot(v) + if dir == modRoot { + return mms.PathPrefix(v), v + } + if str.HasFilePathPrefix(dir, modRoot) { + pathPrefix := MainModules.PathPrefix(v) + if pathPrefix > longestPrefix { + longestPrefix = pathPrefix + longestPrefixVersion = v + suffix := filepath.ToSlash(str.TrimFilePathPrefix(dir, modRoot)) + if strings.HasPrefix(suffix, "vendor/") { + longestPrefixPath = suffix[len("vendor/"):] + continue + } + longestPrefixPath = pathpkg.Join(mms.PathPrefix(v), suffix) + } + } + } + if len(longestPrefix) > 0 { + return longestPrefixPath, longestPrefixVersion + } + + return ".", module.Version{} +} + +// PackageModule returns the module providing the package named by the import path. +func PackageModule(path string) module.Version { + pkg, ok := loaded.pkgCache.Get(path) + if !ok { + return module.Version{} + } + return pkg.mod +} + +// Lookup returns the source directory, import path, and any loading error for +// the package at path as imported from the package in parentDir. +// Lookup requires that one of the Load functions in this package has already +// been called. +func Lookup(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) { + if path == "" { + panic("Lookup called with empty package path") + } + + if parentIsStd { + path = loaded.stdVendor(parentPath, path) + } + pkg, ok := loaded.pkgCache.Get(path) + if !ok { + // The loader should have found all the relevant paths. + // There are a few exceptions, though: + // - during go list without -test, the p.Resolve calls to process p.TestImports and p.XTestImports + // end up here to canonicalize the import paths. + // - during any load, non-loaded packages like "unsafe" end up here. + // - during any load, build-injected dependencies like "runtime/cgo" end up here. + // - because we ignore appengine/* in the module loader, + // the dependencies of any actual appengine/* library end up here. + dir := findStandardImportPath(path) + if dir != "" { + return dir, path, nil + } + return "", "", errMissing + } + return pkg.dir, pkg.path, pkg.err +} + +// A loader manages the process of loading information about +// the required packages for a particular build, +// checking that the packages are available in the module set, +// and updating the module set if needed. +type loader struct { + loaderParams + + // allClosesOverTests indicates whether the "all" pattern includes + // dependencies of tests outside the main module (as in Go 1.11–1.15). + // (Otherwise — as in Go 1.16+ — the "all" pattern includes only the packages + // transitively *imported by* the packages and tests in the main module.) + allClosesOverTests bool + + // skipImportModFiles indicates whether we may skip loading go.mod files + // for imported packages (as in 'go mod tidy' in Go 1.17–1.20). + skipImportModFiles bool + + work *par.Queue + + // reset on each iteration + roots []*loadPkg + pkgCache *par.Cache[string, *loadPkg] + pkgs []*loadPkg // transitive closure of loaded packages and tests; populated in buildStacks +} + +// loaderParams configure the packages loaded by, and the properties reported +// by, a loader instance. +type loaderParams struct { + PackageOpts + requirements *Requirements + + allPatternIsRoot bool // Is the "all" pattern an additional root? + + listRoots func(rs *Requirements) []string +} + +func (ld *loader) reset() { + select { + case <-ld.work.Idle(): + default: + panic("loader.reset when not idle") + } + + ld.roots = nil + ld.pkgCache = new(par.Cache[string, *loadPkg]) + ld.pkgs = nil +} + +// error reports an error via either os.Stderr or base.Error, +// according to whether ld.AllowErrors is set. +func (ld *loader) error(err error) { + if ld.AllowErrors { + fmt.Fprintf(os.Stderr, "go: %v\n", err) + } else if ld.Switcher != nil { + ld.Switcher.Error(err) + } else { + base.Error(err) + } +} + +// switchIfErrors switches toolchains if a switch is needed. +func (ld *loader) switchIfErrors(ctx context.Context) { + if ld.Switcher != nil { + ld.Switcher.Switch(ctx) + } +} + +// exitIfErrors switches toolchains if a switch is needed +// or else exits if any errors have been reported. +func (ld *loader) exitIfErrors(ctx context.Context) { + ld.switchIfErrors(ctx) + base.ExitIfErrors() +} + +// goVersion reports the Go version that should be used for the loader's +// requirements: ld.TidyGoVersion if set, or ld.requirements.GoVersion() +// otherwise. +func (ld *loader) goVersion() string { + if ld.TidyGoVersion != "" { + return ld.TidyGoVersion + } + return ld.requirements.GoVersion() +} + +// A loadPkg records information about a single loaded package. +type loadPkg struct { + // Populated at construction time: + path string // import path + testOf *loadPkg + + // Populated at construction time and updated by (*loader).applyPkgFlags: + flags atomicLoadPkgFlags + + // Populated by (*loader).load: + mod module.Version // module providing package + dir string // directory containing source code + err error // error loading package + imports []*loadPkg // packages imported by this one + testImports []string // test-only imports, saved for use by pkg.test. + inStd bool + altMods []module.Version // modules that could have contained the package but did not + + // Populated by (*loader).pkgTest: + testOnce sync.Once + test *loadPkg + + // Populated by postprocessing in (*loader).buildStacks: + stack *loadPkg // package importing this one in minimal import stack for this pkg +} + +// loadPkgFlags is a set of flags tracking metadata about a package. +type loadPkgFlags int8 + +const ( + // pkgInAll indicates that the package is in the "all" package pattern, + // regardless of whether we are loading the "all" package pattern. + // + // When the pkgInAll flag and pkgImportsLoaded flags are both set, the caller + // who set the last of those flags must propagate the pkgInAll marking to all + // of the imports of the marked package. + // + // A test is marked with pkgInAll if that test would promote the packages it + // imports to be in "all" (such as when the test is itself within the main + // module, or when ld.allClosesOverTests is true). + pkgInAll loadPkgFlags = 1 << iota + + // pkgIsRoot indicates that the package matches one of the root package + // patterns requested by the caller. + // + // If LoadTests is set, then when pkgIsRoot and pkgImportsLoaded are both set, + // the caller who set the last of those flags must populate a test for the + // package (in the pkg.test field). + // + // If the "all" pattern is included as a root, then non-test packages in "all" + // are also roots (and must be marked pkgIsRoot). + pkgIsRoot + + // pkgFromRoot indicates that the package is in the transitive closure of + // imports starting at the roots. (Note that every package marked as pkgIsRoot + // is also trivially marked pkgFromRoot.) + pkgFromRoot + + // pkgImportsLoaded indicates that the imports and testImports fields of a + // loadPkg have been populated. + pkgImportsLoaded +) + +// has reports whether all of the flags in cond are set in f. +func (f loadPkgFlags) has(cond loadPkgFlags) bool { + return f&cond == cond +} + +// An atomicLoadPkgFlags stores a loadPkgFlags for which individual flags can be +// added atomically. +type atomicLoadPkgFlags struct { + bits atomic.Int32 +} + +// update sets the given flags in af (in addition to any flags already set). +// +// update returns the previous flag state so that the caller may determine which +// flags were newly-set. +func (af *atomicLoadPkgFlags) update(flags loadPkgFlags) (old loadPkgFlags) { + for { + old := af.bits.Load() + new := old | int32(flags) + if new == old || af.bits.CompareAndSwap(old, new) { + return loadPkgFlags(old) + } + } +} + +// has reports whether all of the flags in cond are set in af. +func (af *atomicLoadPkgFlags) has(cond loadPkgFlags) bool { + return loadPkgFlags(af.bits.Load())&cond == cond +} + +// isTest reports whether pkg is a test of another package. +func (pkg *loadPkg) isTest() bool { + return pkg.testOf != nil +} + +// fromExternalModule reports whether pkg was loaded from a module other than +// the main module. +func (pkg *loadPkg) fromExternalModule() bool { + if pkg.mod.Path == "" { + return false // loaded from the standard library, not a module + } + return !MainModules.Contains(pkg.mod.Path) +} + +var errMissing = errors.New("cannot find package") + +// loadFromRoots attempts to load the build graph needed to process a set of +// root packages and their dependencies. +// +// The set of root packages is returned by the params.listRoots function, and +// expanded to the full set of packages by tracing imports (and possibly tests) +// as needed. +func loadFromRoots(ctx context.Context, params loaderParams) *loader { + ld := &loader{ + loaderParams: params, + work: par.NewQueue(runtime.GOMAXPROCS(0)), + } + + if ld.requirements.pruning == unpruned { + // If the module graph does not support pruning, we assume that we will need + // the full module graph in order to load package dependencies. + // + // This might not be strictly necessary, but it matches the historical + // behavior of the 'go' command and keeps the go.mod file more consistent in + // case of erroneous hand-edits — which are less likely to be detected by + // spot-checks in modules that do not maintain the expanded go.mod + // requirements needed for graph pruning. + var err error + ld.requirements, _, err = expandGraph(ctx, ld.requirements) + if err != nil { + ld.error(err) + } + } + ld.exitIfErrors(ctx) + + updateGoVersion := func() { + goVersion := ld.goVersion() + + if ld.requirements.pruning != workspace { + var err error + ld.requirements, err = convertPruning(ctx, ld.requirements, pruningForGoVersion(goVersion)) + if err != nil { + ld.error(err) + ld.exitIfErrors(ctx) + } + } + + // If the module's Go version omits go.sum entries for go.mod files for test + // dependencies of external packages, avoid loading those files in the first + // place. + ld.skipImportModFiles = ld.Tidy && gover.Compare(goVersion, gover.TidyGoModSumVersion) < 0 + + // If the module's go version explicitly predates the change in "all" for + // graph pruning, continue to use the older interpretation. + ld.allClosesOverTests = gover.Compare(goVersion, gover.NarrowAllVersion) < 0 && !ld.UseVendorAll + } + + for { + ld.reset() + updateGoVersion() + + // Load the root packages and their imports. + // Note: the returned roots can change on each iteration, + // since the expansion of package patterns depends on the + // build list we're using. + rootPkgs := ld.listRoots(ld.requirements) + + if ld.requirements.pruning == pruned && cfg.BuildMod == "mod" { + // Before we start loading transitive imports of packages, locate all of + // the root packages and promote their containing modules to root modules + // dependencies. If their go.mod files are tidy (the common case) and the + // set of root packages does not change then we can select the correct + // versions of all transitive imports on the first try and complete + // loading in a single iteration. + changedBuildList := ld.preloadRootModules(ctx, rootPkgs) + if changedBuildList { + // The build list has changed, so the set of root packages may have also + // changed. Start over to pick up the changes. (Preloading roots is much + // cheaper than loading the full import graph, so we would rather pay + // for an extra iteration of preloading than potentially end up + // discarding the result of a full iteration of loading.) + continue + } + } + + inRoots := map[*loadPkg]bool{} + for _, path := range rootPkgs { + root := ld.pkg(ctx, path, pkgIsRoot) + if !inRoots[root] { + ld.roots = append(ld.roots, root) + inRoots[root] = true + } + } + + // ld.pkg adds imported packages to the work queue and calls applyPkgFlags, + // which adds tests (and test dependencies) as needed. + // + // When all of the work in the queue has completed, we'll know that the + // transitive closure of dependencies has been loaded. + <-ld.work.Idle() + + ld.buildStacks() + + changed, err := ld.updateRequirements(ctx) + if err != nil { + ld.error(err) + break + } + if changed { + // Don't resolve missing imports until the module graph has stabilized. + // If the roots are still changing, they may turn out to specify a + // requirement on the missing package(s), and we would rather use a + // version specified by a new root than add a new dependency on an + // unrelated version. + continue + } + + if !ld.ResolveMissingImports || (!HasModRoot() && !allowMissingModuleImports) { + // We've loaded as much as we can without resolving missing imports. + break + } + + modAddedBy, err := ld.resolveMissingImports(ctx) + if err != nil { + ld.error(err) + break + } + if len(modAddedBy) == 0 { + // The roots are stable, and we've resolved all of the missing packages + // that we can. + break + } + + toAdd := make([]module.Version, 0, len(modAddedBy)) + for m := range modAddedBy { + toAdd = append(toAdd, m) + } + gover.ModSort(toAdd) // to make errors deterministic + + // We ran updateRequirements before resolving missing imports and it didn't + // make any changes, so we know that the requirement graph is already + // consistent with ld.pkgs: we don't need to pass ld.pkgs to updateRoots + // again. (That would waste time looking for changes that we have already + // applied.) + var noPkgs []*loadPkg + // We also know that we're going to call updateRequirements again next + // iteration so we don't need to also update it here. (That would waste time + // computing a "direct" map that we'll have to recompute later anyway.) + direct := ld.requirements.direct + rs, err := updateRoots(ctx, direct, ld.requirements, noPkgs, toAdd, ld.AssumeRootsImported) + if err != nil { + // If an error was found in a newly added module, report the package + // import stack instead of the module requirement stack. Packages + // are more descriptive. + if err, ok := err.(*mvs.BuildListError); ok { + if pkg := modAddedBy[err.Module()]; pkg != nil { + ld.error(fmt.Errorf("%s: %w", pkg.stackText(), err.Err)) + break + } + } + ld.error(err) + break + } + if reflect.DeepEqual(rs.rootModules, ld.requirements.rootModules) { + // Something is deeply wrong. resolveMissingImports gave us a non-empty + // set of modules to add to the graph, but adding those modules had no + // effect — either they were already in the graph, or updateRoots did not + // add them as requested. + panic(fmt.Sprintf("internal error: adding %v to module graph had no effect on root requirements (%v)", toAdd, rs.rootModules)) + } + ld.requirements = rs + } + ld.exitIfErrors(ctx) + + // Tidy the build list, if applicable, before we report errors. + // (The process of tidying may remove errors from irrelevant dependencies.) + if ld.Tidy { + rs, err := tidyRoots(ctx, ld.requirements, ld.pkgs) + if err != nil { + ld.error(err) + } else { + if ld.TidyGoVersion != "" { + // Attempt to switch to the requested Go version. We have been using its + // pruning and semantics all along, but there may have been — and may + // still be — requirements on higher versions in the graph. + tidy := overrideRoots(ctx, rs, []module.Version{{Path: "go", Version: ld.TidyGoVersion}}) + mg, err := tidy.Graph(ctx) + if err != nil { + ld.error(err) + } + if v := mg.Selected("go"); v == ld.TidyGoVersion { + rs = tidy + } else { + conflict := Conflict{ + Path: mg.g.FindPath(func(m module.Version) bool { + return m.Path == "go" && m.Version == v + })[1:], + Constraint: module.Version{Path: "go", Version: ld.TidyGoVersion}, + } + msg := conflict.Summary() + if cfg.BuildV { + msg = conflict.String() + } + ld.error(errors.New(msg)) + } + } + + if ld.requirements.pruning == pruned { + // We continuously add tidy roots to ld.requirements during loading, so + // at this point the tidy roots (other than possibly the "go" version + // edited above) should be a subset of the roots of ld.requirements, + // ensuring that no new dependencies are brought inside the + // graph-pruning horizon. + // If that is not the case, there is a bug in the loading loop above. + for _, m := range rs.rootModules { + if m.Path == "go" && ld.TidyGoVersion != "" { + continue + } + if v, ok := ld.requirements.rootSelected(m.Path); !ok || v != m.Version { + ld.error(fmt.Errorf("internal error: a requirement on %v is needed but was not added during package loading (selected %s)", m, v)) + } + } + } + + ld.requirements = rs + } + + ld.exitIfErrors(ctx) + } + + // Report errors, if any. + for _, pkg := range ld.pkgs { + if pkg.err == nil { + continue + } + + // Add importer information to checksum errors. + if sumErr := (*ImportMissingSumError)(nil); errors.As(pkg.err, &sumErr) { + if importer := pkg.stack; importer != nil { + sumErr.importer = importer.path + sumErr.importerVersion = importer.mod.Version + sumErr.importerIsTest = importer.testOf != nil + } + } + + if stdErr := (*ImportMissingError)(nil); errors.As(pkg.err, &stdErr) && stdErr.isStd { + // Add importer go version information to import errors of standard + // library packages arising from newer releases. + if importer := pkg.stack; importer != nil { + if v, ok := rawGoVersion.Load(importer.mod); ok && gover.Compare(gover.Local(), v.(string)) < 0 { + stdErr.importerGoVersion = v.(string) + } + } + if ld.SilenceMissingStdImports { + continue + } + } + if ld.SilencePackageErrors { + continue + } + if ld.SilenceNoGoErrors && errors.Is(pkg.err, imports.ErrNoGo) { + continue + } + + ld.error(fmt.Errorf("%s: %w", pkg.stackText(), pkg.err)) + } + + ld.checkMultiplePaths() + return ld +} + +// updateRequirements ensures that ld.requirements is consistent with the +// information gained from ld.pkgs. +// +// In particular: +// +// - Modules that provide packages directly imported from the main module are +// marked as direct, and are promoted to explicit roots. If a needed root +// cannot be promoted due to -mod=readonly or -mod=vendor, the importing +// package is marked with an error. +// +// - If ld scanned the "all" pattern independent of build constraints, it is +// guaranteed to have seen every direct import. Module dependencies that did +// not provide any directly-imported package are then marked as indirect. +// +// - Root dependencies are updated to their selected versions. +// +// The "changed" return value reports whether the update changed the selected +// version of any module that either provided a loaded package or may now +// provide a package that was previously unresolved. +func (ld *loader) updateRequirements(ctx context.Context) (changed bool, err error) { + rs := ld.requirements + + // direct contains the set of modules believed to provide packages directly + // imported by the main module. + var direct map[string]bool + + // If we didn't scan all of the imports from the main module, or didn't use + // imports.AnyTags, then we didn't necessarily load every package that + // contributes “direct” imports — so we can't safely mark existing direct + // dependencies in ld.requirements as indirect-only. Propagate them as direct. + loadedDirect := ld.allPatternIsRoot && reflect.DeepEqual(ld.Tags, imports.AnyTags()) + if loadedDirect { + direct = make(map[string]bool) + } else { + // TODO(bcmills): It seems like a shame to allocate and copy a map here when + // it will only rarely actually vary from rs.direct. Measure this cost and + // maybe avoid the copy. + direct = make(map[string]bool, len(rs.direct)) + for mPath := range rs.direct { + direct[mPath] = true + } + } + + var maxTooNew *gover.TooNewError + for _, pkg := range ld.pkgs { + if pkg.err != nil { + if tooNew := (*gover.TooNewError)(nil); errors.As(pkg.err, &tooNew) { + if maxTooNew == nil || gover.Compare(tooNew.GoVersion, maxTooNew.GoVersion) > 0 { + maxTooNew = tooNew + } + } + } + if pkg.mod.Version != "" || !MainModules.Contains(pkg.mod.Path) { + continue + } + + for _, dep := range pkg.imports { + if !dep.fromExternalModule() { + continue + } + + if inWorkspaceMode() { + // In workspace mode / workspace pruning mode, the roots are the main modules + // rather than the main module's direct dependencies. The check below on the selected + // roots does not apply. + if cfg.BuildMod == "vendor" { + // In workspace vendor mode, we don't need to load the requirements of the workspace + // modules' dependencies so the check below doesn't work. But that's okay, because + // checking whether modules are required directly for the purposes of pruning is + // less important in vendor mode: if we were able to load the package, we have + // everything we need to build the package, and dependencies' tests are pruned out + // of the vendor directory anyway. + continue + } + if mg, err := rs.Graph(ctx); err != nil { + return false, err + } else if _, ok := mg.RequiredBy(dep.mod); !ok { + // dep.mod is not an explicit dependency, but needs to be. + // See comment on error returned below. + pkg.err = &DirectImportFromImplicitDependencyError{ + ImporterPath: pkg.path, + ImportedPath: dep.path, + Module: dep.mod, + } + } + continue + } + + if pkg.err == nil && cfg.BuildMod != "mod" { + if v, ok := rs.rootSelected(dep.mod.Path); !ok || v != dep.mod.Version { + // dep.mod is not an explicit dependency, but needs to be. + // Because we are not in "mod" mode, we will not be able to update it. + // Instead, mark the importing package with an error. + // + // TODO(#41688): The resulting error message fails to include the file + // position of the import statement (because that information is not + // tracked by the module loader). Figure out how to plumb the import + // position through. + pkg.err = &DirectImportFromImplicitDependencyError{ + ImporterPath: pkg.path, + ImportedPath: dep.path, + Module: dep.mod, + } + // cfg.BuildMod does not allow us to change dep.mod to be a direct + // dependency, so don't mark it as such. + continue + } + } + + // dep is a package directly imported by a package or test in the main + // module and loaded from some other module (not the standard library). + // Mark its module as a direct dependency. + direct[dep.mod.Path] = true + } + } + if maxTooNew != nil { + return false, maxTooNew + } + + var addRoots []module.Version + if ld.Tidy { + // When we are tidying a module with a pruned dependency graph, we may need + // to add roots to preserve the versions of indirect, test-only dependencies + // that are upgraded above or otherwise missing from the go.mod files of + // direct dependencies. (For example, the direct dependency might be a very + // stable codebase that predates modules and thus lacks a go.mod file, or + // the author of the direct dependency may have forgotten to commit a change + // to the go.mod file, or may have made an erroneous hand-edit that causes + // it to be untidy.) + // + // Promoting an indirect dependency to a root adds the next layer of its + // dependencies to the module graph, which may increase the selected + // versions of other modules from which we have already loaded packages. + // So after we promote an indirect dependency to a root, we need to reload + // packages, which means another iteration of loading. + // + // As an extra wrinkle, the upgrades due to promoting a root can cause + // previously-resolved packages to become unresolved. For example, the + // module providing an unstable package might be upgraded to a version + // that no longer contains that package. If we then resolve the missing + // package, we might add yet another root that upgrades away some other + // dependency. (The tests in mod_tidy_convergence*.txt illustrate some + // particularly worrisome cases.) + // + // To ensure that this process of promoting, adding, and upgrading roots + // eventually terminates, during iteration we only ever add modules to the + // root set — we only remove irrelevant roots at the very end of + // iteration, after we have already added every root that we plan to need + // in the (eventual) tidy root set. + // + // Since we do not remove any roots during iteration, even if they no + // longer provide any imported packages, the selected versions of the + // roots can only increase and the set of roots can only expand. The set + // of extant root paths is finite and the set of versions of each path is + // finite, so the iteration *must* reach a stable fixed-point. + tidy, err := tidyRoots(ctx, rs, ld.pkgs) + if err != nil { + return false, err + } + addRoots = tidy.rootModules + } + + rs, err = updateRoots(ctx, direct, rs, ld.pkgs, addRoots, ld.AssumeRootsImported) + if err != nil { + // We don't actually know what even the root requirements are supposed to be, + // so we can't proceed with loading. Return the error to the caller + return false, err + } + + if rs.GoVersion() != ld.requirements.GoVersion() { + // A change in the selected Go version may or may not affect the set of + // loaded packages, but in some cases it can change the meaning of the "all" + // pattern, the level of pruning in the module graph, and even the set of + // packages present in the standard library. If it has changed, it's best to + // reload packages once more to be sure everything is stable. + changed = true + } else if rs != ld.requirements && !reflect.DeepEqual(rs.rootModules, ld.requirements.rootModules) { + // The roots of the module graph have changed in some way (not just the + // "direct" markings). Check whether the changes affected any of the loaded + // packages. + mg, err := rs.Graph(ctx) + if err != nil { + return false, err + } + for _, pkg := range ld.pkgs { + if pkg.fromExternalModule() && mg.Selected(pkg.mod.Path) != pkg.mod.Version { + changed = true + break + } + if pkg.err != nil { + // Promoting a module to a root may resolve an import that was + // previously missing (by pulling in a previously-prune dependency that + // provides it) or ambiguous (by promoting exactly one of the + // alternatives to a root and ignoring the second-level alternatives) or + // otherwise errored out (by upgrading from a version that cannot be + // fetched to one that can be). + // + // Instead of enumerating all of the possible errors, we'll just check + // whether importFromModules returns nil for the package. + // False-positives are ok: if we have a false-positive here, we'll do an + // extra iteration of package loading this time, but we'll still + // converge when the root set stops changing. + // + // In some sense, we can think of this as ‘upgraded the module providing + // pkg.path from "none" to a version higher than "none"’. + if _, _, _, _, err = importFromModules(ctx, pkg.path, rs, nil, ld.skipImportModFiles); err == nil { + changed = true + break + } + } + } + } + + ld.requirements = rs + return changed, nil +} + +// resolveMissingImports returns a set of modules that could be added as +// dependencies in order to resolve missing packages from pkgs. +// +// The newly-resolved packages are added to the addedModuleFor map, and +// resolveMissingImports returns a map from each new module version to +// the first missing package that module would resolve. +func (ld *loader) resolveMissingImports(ctx context.Context) (modAddedBy map[module.Version]*loadPkg, err error) { + type pkgMod struct { + pkg *loadPkg + mod *module.Version + } + var pkgMods []pkgMod + for _, pkg := range ld.pkgs { + if pkg.err == nil { + continue + } + if pkg.isTest() { + // If we are missing a test, we are also missing its non-test version, and + // we should only add the missing import once. + continue + } + if !errors.As(pkg.err, new(*ImportMissingError)) { + // Leave other errors for Import or load.Packages to report. + continue + } + + pkg := pkg + var mod module.Version + ld.work.Add(func() { + var err error + mod, err = queryImport(ctx, pkg.path, ld.requirements) + if err != nil { + var ime *ImportMissingError + if errors.As(err, &ime) { + for curstack := pkg.stack; curstack != nil; curstack = curstack.stack { + if MainModules.Contains(curstack.mod.Path) { + ime.ImportingMainModule = curstack.mod + break + } + } + } + // pkg.err was already non-nil, so we can reasonably attribute the error + // for pkg to either the original error or the one returned by + // queryImport. The existing error indicates only that we couldn't find + // the package, whereas the query error also explains why we didn't fix + // the problem — so we prefer the latter. + pkg.err = err + } + + // err is nil, but we intentionally leave pkg.err non-nil and pkg.mod + // unset: we still haven't satisfied other invariants of a + // successfully-loaded package, such as scanning and loading the imports + // of that package. If we succeed in resolving the new dependency graph, + // the caller can reload pkg and update the error at that point. + // + // Even then, the package might not be loaded from the version we've + // identified here. The module may be upgraded by some other dependency, + // or by a transitive dependency of mod itself, or — less likely — the + // package may be rejected by an AllowPackage hook or rendered ambiguous + // by some other newly-added or newly-upgraded dependency. + }) + + pkgMods = append(pkgMods, pkgMod{pkg: pkg, mod: &mod}) + } + <-ld.work.Idle() + + modAddedBy = map[module.Version]*loadPkg{} + + var ( + maxTooNew *gover.TooNewError + maxTooNewPkg *loadPkg + ) + for _, pm := range pkgMods { + if tooNew := (*gover.TooNewError)(nil); errors.As(pm.pkg.err, &tooNew) { + if maxTooNew == nil || gover.Compare(tooNew.GoVersion, maxTooNew.GoVersion) > 0 { + maxTooNew = tooNew + maxTooNewPkg = pm.pkg + } + } + } + if maxTooNew != nil { + fmt.Fprintf(os.Stderr, "go: toolchain upgrade needed to resolve %s\n", maxTooNewPkg.path) + return nil, maxTooNew + } + + for _, pm := range pkgMods { + pkg, mod := pm.pkg, *pm.mod + if mod.Path == "" { + continue + } + + fmt.Fprintf(os.Stderr, "go: found %s in %s %s\n", pkg.path, mod.Path, mod.Version) + if modAddedBy[mod] == nil { + modAddedBy[mod] = pkg + } + } + + return modAddedBy, nil +} + +// pkg locates the *loadPkg for path, creating and queuing it for loading if +// needed, and updates its state to reflect the given flags. +// +// The imports of the returned *loadPkg will be loaded asynchronously in the +// ld.work queue, and its test (if requested) will also be populated once +// imports have been resolved. When ld.work goes idle, all transitive imports of +// the requested package (and its test, if requested) will have been loaded. +func (ld *loader) pkg(ctx context.Context, path string, flags loadPkgFlags) *loadPkg { + if flags.has(pkgImportsLoaded) { + panic("internal error: (*loader).pkg called with pkgImportsLoaded flag set") + } + + pkg := ld.pkgCache.Do(path, func() *loadPkg { + pkg := &loadPkg{ + path: path, + } + ld.applyPkgFlags(ctx, pkg, flags) + + ld.work.Add(func() { ld.load(ctx, pkg) }) + return pkg + }) + + ld.applyPkgFlags(ctx, pkg, flags) + return pkg +} + +// applyPkgFlags updates pkg.flags to set the given flags and propagate the +// (transitive) effects of those flags, possibly loading or enqueueing further +// packages as a result. +func (ld *loader) applyPkgFlags(ctx context.Context, pkg *loadPkg, flags loadPkgFlags) { + if flags == 0 { + return + } + + if flags.has(pkgInAll) && ld.allPatternIsRoot && !pkg.isTest() { + // This package matches a root pattern by virtue of being in "all". + flags |= pkgIsRoot + } + if flags.has(pkgIsRoot) { + flags |= pkgFromRoot + } + + old := pkg.flags.update(flags) + new := old | flags + if new == old || !new.has(pkgImportsLoaded) { + // We either didn't change the state of pkg, or we don't know anything about + // its dependencies yet. Either way, we can't usefully load its test or + // update its dependencies. + return + } + + if !pkg.isTest() { + // Check whether we should add (or update the flags for) a test for pkg. + // ld.pkgTest is idempotent and extra invocations are inexpensive, + // so it's ok if we call it more than is strictly necessary. + wantTest := false + switch { + case ld.allPatternIsRoot && MainModules.Contains(pkg.mod.Path): + // We are loading the "all" pattern, which includes packages imported by + // tests in the main module. This package is in the main module, so we + // need to identify the imports of its test even if LoadTests is not set. + // + // (We will filter out the extra tests explicitly in computePatternAll.) + wantTest = true + + case ld.allPatternIsRoot && ld.allClosesOverTests && new.has(pkgInAll): + // This variant of the "all" pattern includes imports of tests of every + // package that is itself in "all", and pkg is in "all", so its test is + // also in "all" (as above). + wantTest = true + + case ld.LoadTests && new.has(pkgIsRoot): + // LoadTest explicitly requests tests of “the root packages”. + wantTest = true + } + + if wantTest { + var testFlags loadPkgFlags + if MainModules.Contains(pkg.mod.Path) || (ld.allClosesOverTests && new.has(pkgInAll)) { + // Tests of packages in the main module are in "all", in the sense that + // they cause the packages they import to also be in "all". So are tests + // of packages in "all" if "all" closes over test dependencies. + testFlags |= pkgInAll + } + ld.pkgTest(ctx, pkg, testFlags) + } + } + + if new.has(pkgInAll) && !old.has(pkgInAll|pkgImportsLoaded) { + // We have just marked pkg with pkgInAll, or we have just loaded its + // imports, or both. Now is the time to propagate pkgInAll to the imports. + for _, dep := range pkg.imports { + ld.applyPkgFlags(ctx, dep, pkgInAll) + } + } + + if new.has(pkgFromRoot) && !old.has(pkgFromRoot|pkgImportsLoaded) { + for _, dep := range pkg.imports { + ld.applyPkgFlags(ctx, dep, pkgFromRoot) + } + } +} + +// preloadRootModules loads the module requirements needed to identify the +// selected version of each module providing a package in rootPkgs, +// adding new root modules to the module graph if needed. +func (ld *loader) preloadRootModules(ctx context.Context, rootPkgs []string) (changedBuildList bool) { + needc := make(chan map[module.Version]bool, 1) + needc <- map[module.Version]bool{} + for _, path := range rootPkgs { + path := path + ld.work.Add(func() { + // First, try to identify the module containing the package using only roots. + // + // If the main module is tidy and the package is in "all" — or if we're + // lucky — we can identify all of its imports without actually loading the + // full module graph. + m, _, _, _, err := importFromModules(ctx, path, ld.requirements, nil, ld.skipImportModFiles) + if err != nil { + var missing *ImportMissingError + if errors.As(err, &missing) && ld.ResolveMissingImports { + // This package isn't provided by any selected module. + // If we can find it, it will be a new root dependency. + m, err = queryImport(ctx, path, ld.requirements) + } + if err != nil { + // We couldn't identify the root module containing this package. + // Leave it unresolved; we will report it during loading. + return + } + } + if m.Path == "" { + // The package is in std or cmd. We don't need to change the root set. + return + } + + v, ok := ld.requirements.rootSelected(m.Path) + if !ok || v != m.Version { + // We found the requested package in m, but m is not a root, so + // loadModGraph will not load its requirements. We need to promote the + // module to a root to ensure that any other packages this package + // imports are resolved from correct dependency versions. + // + // (This is the “argument invariant” from + // https://golang.org/design/36460-lazy-module-loading.) + need := <-needc + need[m] = true + needc <- need + } + }) + } + <-ld.work.Idle() + + need := <-needc + if len(need) == 0 { + return false // No roots to add. + } + + toAdd := make([]module.Version, 0, len(need)) + for m := range need { + toAdd = append(toAdd, m) + } + gover.ModSort(toAdd) + + rs, err := updateRoots(ctx, ld.requirements.direct, ld.requirements, nil, toAdd, ld.AssumeRootsImported) + if err != nil { + // We are missing some root dependency, and for some reason we can't load + // enough of the module dependency graph to add the missing root. Package + // loading is doomed to fail, so fail quickly. + ld.error(err) + ld.exitIfErrors(ctx) + return false + } + if reflect.DeepEqual(rs.rootModules, ld.requirements.rootModules) { + // Something is deeply wrong. resolveMissingImports gave us a non-empty + // set of modules to add to the graph, but adding those modules had no + // effect — either they were already in the graph, or updateRoots did not + // add them as requested. + panic(fmt.Sprintf("internal error: adding %v to module graph had no effect on root requirements (%v)", toAdd, rs.rootModules)) + } + + ld.requirements = rs + return true +} + +// load loads an individual package. +func (ld *loader) load(ctx context.Context, pkg *loadPkg) { + var mg *ModuleGraph + if ld.requirements.pruning == unpruned { + var err error + mg, err = ld.requirements.Graph(ctx) + if err != nil { + // We already checked the error from Graph in loadFromRoots and/or + // updateRequirements, so we ignored the error on purpose and we should + // keep trying to push past it. + // + // However, because mg may be incomplete (and thus may select inaccurate + // versions), we shouldn't use it to load packages. Instead, we pass a nil + // *ModuleGraph, which will cause mg to first try loading from only the + // main module and root dependencies. + mg = nil + } + } + + var modroot string + pkg.mod, modroot, pkg.dir, pkg.altMods, pkg.err = importFromModules(ctx, pkg.path, ld.requirements, mg, ld.skipImportModFiles) + if pkg.dir == "" { + return + } + if MainModules.Contains(pkg.mod.Path) { + // Go ahead and mark pkg as in "all". This provides the invariant that a + // package that is *only* imported by other packages in "all" is always + // marked as such before loading its imports. + // + // We don't actually rely on that invariant at the moment, but it may + // improve efficiency somewhat and makes the behavior a bit easier to reason + // about (by reducing churn on the flag bits of dependencies), and costs + // essentially nothing (these atomic flag ops are essentially free compared + // to scanning source code for imports). + ld.applyPkgFlags(ctx, pkg, pkgInAll) + } + if ld.AllowPackage != nil { + if err := ld.AllowPackage(ctx, pkg.path, pkg.mod); err != nil { + pkg.err = err + } + } + + pkg.inStd = (search.IsStandardImportPath(pkg.path) && search.InDir(pkg.dir, cfg.GOROOTsrc) != "") + + var imports, testImports []string + + if cfg.BuildContext.Compiler == "gccgo" && pkg.inStd { + // We can't scan standard packages for gccgo. + } else { + var err error + imports, testImports, err = scanDir(modroot, pkg.dir, ld.Tags) + if err != nil { + pkg.err = err + return + } + } + + pkg.imports = make([]*loadPkg, 0, len(imports)) + var importFlags loadPkgFlags + if pkg.flags.has(pkgInAll) { + importFlags = pkgInAll + } + for _, path := range imports { + if pkg.inStd { + // Imports from packages in "std" and "cmd" should resolve using + // GOROOT/src/vendor even when "std" is not the main module. + path = ld.stdVendor(pkg.path, path) + } + pkg.imports = append(pkg.imports, ld.pkg(ctx, path, importFlags)) + } + pkg.testImports = testImports + + ld.applyPkgFlags(ctx, pkg, pkgImportsLoaded) +} + +// pkgTest locates the test of pkg, creating it if needed, and updates its state +// to reflect the given flags. +// +// pkgTest requires that the imports of pkg have already been loaded (flagged +// with pkgImportsLoaded). +func (ld *loader) pkgTest(ctx context.Context, pkg *loadPkg, testFlags loadPkgFlags) *loadPkg { + if pkg.isTest() { + panic("pkgTest called on a test package") + } + + createdTest := false + pkg.testOnce.Do(func() { + pkg.test = &loadPkg{ + path: pkg.path, + testOf: pkg, + mod: pkg.mod, + dir: pkg.dir, + err: pkg.err, + inStd: pkg.inStd, + } + ld.applyPkgFlags(ctx, pkg.test, testFlags) + createdTest = true + }) + + test := pkg.test + if createdTest { + test.imports = make([]*loadPkg, 0, len(pkg.testImports)) + var importFlags loadPkgFlags + if test.flags.has(pkgInAll) { + importFlags = pkgInAll + } + for _, path := range pkg.testImports { + if pkg.inStd { + path = ld.stdVendor(test.path, path) + } + test.imports = append(test.imports, ld.pkg(ctx, path, importFlags)) + } + pkg.testImports = nil + ld.applyPkgFlags(ctx, test, pkgImportsLoaded) + } else { + ld.applyPkgFlags(ctx, test, testFlags) + } + + return test +} + +// stdVendor returns the canonical import path for the package with the given +// path when imported from the standard-library package at parentPath. +func (ld *loader) stdVendor(parentPath, path string) string { + if search.IsStandardImportPath(path) { + return path + } + + if str.HasPathPrefix(parentPath, "cmd") { + if !ld.VendorModulesInGOROOTSrc || !MainModules.Contains("cmd") { + vendorPath := pathpkg.Join("cmd", "vendor", path) + + if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { + return vendorPath + } + } + } else if !ld.VendorModulesInGOROOTSrc || !MainModules.Contains("std") || str.HasPathPrefix(parentPath, "vendor") { + // If we are outside of the 'std' module, resolve imports from within 'std' + // to the vendor directory. + // + // Do the same for importers beginning with the prefix 'vendor/' even if we + // are *inside* of the 'std' module: the 'vendor/' packages that resolve + // globally from GOROOT/src/vendor (and are listed as part of 'go list std') + // are distinct from the real module dependencies, and cannot import + // internal packages from the real module. + // + // (Note that although the 'vendor/' packages match the 'std' *package* + // pattern, they are not part of the std *module*, and do not affect + // 'go mod tidy' and similar module commands when working within std.) + vendorPath := pathpkg.Join("vendor", path) + if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { + return vendorPath + } + } + + // Not vendored: resolve from modules. + return path +} + +// computePatternAll returns the list of packages matching pattern "all", +// starting with a list of the import paths for the packages in the main module. +func (ld *loader) computePatternAll() (all []string) { + for _, pkg := range ld.pkgs { + if pkg.flags.has(pkgInAll) && !pkg.isTest() { + all = append(all, pkg.path) + } + } + sort.Strings(all) + return all +} + +// checkMultiplePaths verifies that a given module path is used as itself +// or as a replacement for another module, but not both at the same time. +// +// (See https://golang.org/issue/26607 and https://golang.org/issue/34650.) +func (ld *loader) checkMultiplePaths() { + mods := ld.requirements.rootModules + if cached := ld.requirements.graph.Load(); cached != nil { + if mg := cached.mg; mg != nil { + mods = mg.BuildList() + } + } + + firstPath := map[module.Version]string{} + for _, mod := range mods { + src := resolveReplacement(mod) + if prev, ok := firstPath[src]; !ok { + firstPath[src] = mod.Path + } else if prev != mod.Path { + ld.error(fmt.Errorf("%s@%s used for two different module paths (%s and %s)", src.Path, src.Version, prev, mod.Path)) + } + } +} + +// checkTidyCompatibility emits an error if any package would be loaded from a +// different module under rs than under ld.requirements. +func (ld *loader) checkTidyCompatibility(ctx context.Context, rs *Requirements, compatVersion string) { + goVersion := rs.GoVersion() + suggestUpgrade := false + suggestEFlag := false + suggestFixes := func() { + if ld.AllowErrors { + // The user is explicitly ignoring these errors, so don't bother them with + // other options. + return + } + + // We print directly to os.Stderr because this information is advice about + // how to fix errors, not actually an error itself. + // (The actual errors should have been logged already.) + + fmt.Fprintln(os.Stderr) + + goFlag := "" + if goVersion != MainModules.GoVersion() { + goFlag = " -go=" + goVersion + } + + compatFlag := "" + if compatVersion != gover.Prev(goVersion) { + compatFlag = " -compat=" + compatVersion + } + if suggestUpgrade { + eDesc := "" + eFlag := "" + if suggestEFlag { + eDesc = ", leaving some packages unresolved" + eFlag = " -e" + } + fmt.Fprintf(os.Stderr, "To upgrade to the versions selected by go %s%s:\n\tgo mod tidy%s -go=%s && go mod tidy%s -go=%s%s\n", compatVersion, eDesc, eFlag, compatVersion, eFlag, goVersion, compatFlag) + } else if suggestEFlag { + // If some packages are missing but no package is upgraded, then we + // shouldn't suggest upgrading to the Go 1.16 versions explicitly — that + // wouldn't actually fix anything for Go 1.16 users, and *would* break + // something for Go 1.17 users. + fmt.Fprintf(os.Stderr, "To proceed despite packages unresolved in go %s:\n\tgo mod tidy -e%s%s\n", compatVersion, goFlag, compatFlag) + } + + fmt.Fprintf(os.Stderr, "If reproducibility with go %s is not needed:\n\tgo mod tidy%s -compat=%s\n", compatVersion, goFlag, goVersion) + + // TODO(#46141): Populate the linked wiki page. + fmt.Fprintf(os.Stderr, "For other options, see:\n\thttps://golang.org/doc/modules/pruning\n") + } + + mg, err := rs.Graph(ctx) + if err != nil { + ld.error(fmt.Errorf("error loading go %s module graph: %w", compatVersion, err)) + ld.switchIfErrors(ctx) + suggestFixes() + ld.exitIfErrors(ctx) + return + } + + // Re-resolve packages in parallel. + // + // We re-resolve each package — rather than just checking versions — to ensure + // that we have fetched module source code (and, importantly, checksums for + // that source code) for all modules that are necessary to ensure that imports + // are unambiguous. That also produces clearer diagnostics, since we can say + // exactly what happened to the package if it became ambiguous or disappeared + // entirely. + // + // We re-resolve the packages in parallel because this process involves disk + // I/O to check for package sources, and because the process of checking for + // ambiguous imports may require us to download additional modules that are + // otherwise pruned out in Go 1.17 — we don't want to block progress on other + // packages while we wait for a single new download. + type mismatch struct { + mod module.Version + err error + } + mismatchMu := make(chan map[*loadPkg]mismatch, 1) + mismatchMu <- map[*loadPkg]mismatch{} + for _, pkg := range ld.pkgs { + if pkg.mod.Path == "" && pkg.err == nil { + // This package is from the standard library (which does not vary based on + // the module graph). + continue + } + + pkg := pkg + ld.work.Add(func() { + mod, _, _, _, err := importFromModules(ctx, pkg.path, rs, mg, ld.skipImportModFiles) + if mod != pkg.mod { + mismatches := <-mismatchMu + mismatches[pkg] = mismatch{mod: mod, err: err} + mismatchMu <- mismatches + } + }) + } + <-ld.work.Idle() + + mismatches := <-mismatchMu + if len(mismatches) == 0 { + // Since we're running as part of 'go mod tidy', the roots of the module + // graph should contain only modules that are relevant to some package in + // the package graph. We checked every package in the package graph and + // didn't find any mismatches, so that must mean that all of the roots of + // the module graph are also consistent. + // + // If we're wrong, Go 1.16 in -mod=readonly mode will error out with + // "updates to go.mod needed", which would be very confusing. So instead, + // we'll double-check that our reasoning above actually holds — if it + // doesn't, we'll emit an internal error and hopefully the user will report + // it as a bug. + for _, m := range ld.requirements.rootModules { + if v := mg.Selected(m.Path); v != m.Version { + fmt.Fprintln(os.Stderr) + base.Fatalf("go: internal error: failed to diagnose selected-version mismatch for module %s: go %s selects %s, but go %s selects %s\n\tPlease report this at https://golang.org/issue.", m.Path, goVersion, m.Version, compatVersion, v) + } + } + return + } + + // Iterate over the packages (instead of the mismatches map) to emit errors in + // deterministic order. + for _, pkg := range ld.pkgs { + mismatch, ok := mismatches[pkg] + if !ok { + continue + } + + if pkg.isTest() { + // We already did (or will) report an error for the package itself, + // so don't report a duplicate (and more verbose) error for its test. + if _, ok := mismatches[pkg.testOf]; !ok { + base.Fatalf("go: internal error: mismatch recorded for test %s, but not its non-test package", pkg.path) + } + continue + } + + switch { + case mismatch.err != nil: + // pkg resolved successfully, but errors out using the requirements in rs. + // + // This could occur because the import is provided by a single root (and + // is thus unambiguous in a main module with a pruned module graph) and + // also one or more transitive dependencies (and is ambiguous with an + // unpruned graph). + // + // It could also occur because some transitive dependency upgrades the + // module that previously provided the package to a version that no + // longer does, or to a version for which the module source code (but + // not the go.mod file in isolation) has a checksum error. + if missing := (*ImportMissingError)(nil); errors.As(mismatch.err, &missing) { + selected := module.Version{ + Path: pkg.mod.Path, + Version: mg.Selected(pkg.mod.Path), + } + ld.error(fmt.Errorf("%s loaded from %v,\n\tbut go %s would fail to locate it in %s", pkg.stackText(), pkg.mod, compatVersion, selected)) + } else { + if ambiguous := (*AmbiguousImportError)(nil); errors.As(mismatch.err, &ambiguous) { + // TODO: Is this check needed? + } + ld.error(fmt.Errorf("%s loaded from %v,\n\tbut go %s would fail to locate it:\n\t%v", pkg.stackText(), pkg.mod, compatVersion, mismatch.err)) + } + + suggestEFlag = true + + // Even if we press ahead with the '-e' flag, the older version will + // error out in readonly mode if it thinks the go.mod file contains + // any *explicit* dependency that is not at its selected version, + // even if that dependency is not relevant to any package being loaded. + // + // We check for that condition here. If all of the roots are consistent + // the '-e' flag suffices, but otherwise we need to suggest an upgrade. + if !suggestUpgrade { + for _, m := range ld.requirements.rootModules { + if v := mg.Selected(m.Path); v != m.Version { + suggestUpgrade = true + break + } + } + } + + case pkg.err != nil: + // pkg had an error in with a pruned module graph (presumably suppressed + // with the -e flag), but the error went away using an unpruned graph. + // + // This is possible, if, say, the import is unresolved in the pruned graph + // (because the "latest" version of each candidate module either is + // unavailable or does not contain the package), but is resolved in the + // unpruned graph due to a newer-than-latest dependency that is normally + // pruned out. + // + // This could also occur if the source code for the module providing the + // package in the pruned graph has a checksum error, but the unpruned + // graph upgrades that module to a version with a correct checksum. + // + // pkg.err should have already been logged elsewhere — along with a + // stack trace — so log only the import path and non-error info here. + suggestUpgrade = true + ld.error(fmt.Errorf("%s failed to load from any module,\n\tbut go %s would load it from %v", pkg.path, compatVersion, mismatch.mod)) + + case pkg.mod != mismatch.mod: + // The package is loaded successfully by both Go versions, but from a + // different module in each. This could lead to subtle (and perhaps even + // unnoticed!) variations in behavior between builds with different + // toolchains. + suggestUpgrade = true + ld.error(fmt.Errorf("%s loaded from %v,\n\tbut go %s would select %v\n", pkg.stackText(), pkg.mod, compatVersion, mismatch.mod.Version)) + + default: + base.Fatalf("go: internal error: mismatch recorded for package %s, but no differences found", pkg.path) + } + } + + ld.switchIfErrors(ctx) + suggestFixes() + ld.exitIfErrors(ctx) +} + +// scanDir is like imports.ScanDir but elides known magic imports from the list, +// so that we do not go looking for packages that don't really exist. +// +// The standard magic import is "C", for cgo. +// +// The only other known magic imports are appengine and appengine/*. +// These are so old that they predate "go get" and did not use URL-like paths. +// Most code today now uses google.golang.org/appengine instead, +// but not all code has been so updated. When we mostly ignore build tags +// during "go vendor", we look into "// +build appengine" files and +// may see these legacy imports. We drop them so that the module +// search does not look for modules to try to satisfy them. +func scanDir(modroot string, dir string, tags map[string]bool) (imports_, testImports []string, err error) { + if ip, mierr := modindex.GetPackage(modroot, dir); mierr == nil { + imports_, testImports, err = ip.ScanDir(tags) + goto Happy + } else if !errors.Is(mierr, modindex.ErrNotIndexed) { + return nil, nil, mierr + } + + imports_, testImports, err = imports.ScanDir(dir, tags) +Happy: + + filter := func(x []string) []string { + w := 0 + for _, pkg := range x { + if pkg != "C" && pkg != "appengine" && !strings.HasPrefix(pkg, "appengine/") && + pkg != "appengine_internal" && !strings.HasPrefix(pkg, "appengine_internal/") { + x[w] = pkg + w++ + } + } + return x[:w] + } + + return filter(imports_), filter(testImports), err +} + +// buildStacks computes minimal import stacks for each package, +// for use in error messages. When it completes, packages that +// are part of the original root set have pkg.stack == nil, +// and other packages have pkg.stack pointing at the next +// package up the import stack in their minimal chain. +// As a side effect, buildStacks also constructs ld.pkgs, +// the list of all packages loaded. +func (ld *loader) buildStacks() { + if len(ld.pkgs) > 0 { + panic("buildStacks") + } + for _, pkg := range ld.roots { + pkg.stack = pkg // sentinel to avoid processing in next loop + ld.pkgs = append(ld.pkgs, pkg) + } + for i := 0; i < len(ld.pkgs); i++ { // not range: appending to ld.pkgs in loop + pkg := ld.pkgs[i] + for _, next := range pkg.imports { + if next.stack == nil { + next.stack = pkg + ld.pkgs = append(ld.pkgs, next) + } + } + if next := pkg.test; next != nil && next.stack == nil { + next.stack = pkg + ld.pkgs = append(ld.pkgs, next) + } + } + for _, pkg := range ld.roots { + pkg.stack = nil + } +} + +// stackText builds the import stack text to use when +// reporting an error in pkg. It has the general form +// +// root imports +// other imports +// other2 tested by +// other2.test imports +// pkg +func (pkg *loadPkg) stackText() string { + var stack []*loadPkg + for p := pkg; p != nil; p = p.stack { + stack = append(stack, p) + } + + var buf strings.Builder + for i := len(stack) - 1; i >= 0; i-- { + p := stack[i] + fmt.Fprint(&buf, p.path) + if p.testOf != nil { + fmt.Fprint(&buf, ".test") + } + if i > 0 { + if stack[i-1].testOf == p { + fmt.Fprint(&buf, " tested by\n\t") + } else { + fmt.Fprint(&buf, " imports\n\t") + } + } + } + return buf.String() +} + +// why returns the text to use in "go mod why" output about the given package. +// It is less ornate than the stackText but contains the same information. +func (pkg *loadPkg) why() string { + var buf strings.Builder + var stack []*loadPkg + for p := pkg; p != nil; p = p.stack { + stack = append(stack, p) + } + + for i := len(stack) - 1; i >= 0; i-- { + p := stack[i] + if p.testOf != nil { + fmt.Fprintf(&buf, "%s.test\n", p.testOf.path) + } else { + fmt.Fprintf(&buf, "%s\n", p.path) + } + } + return buf.String() +} + +// Why returns the "go mod why" output stanza for the given package, +// without the leading # comment. +// The package graph must have been loaded already, usually by LoadPackages. +// If there is no reason for the package to be in the current build, +// Why returns an empty string. +func Why(path string) string { + pkg, ok := loaded.pkgCache.Get(path) + if !ok { + return "" + } + return pkg.why() +} + +// WhyDepth returns the number of steps in the Why listing. +// If there is no reason for the package to be in the current build, +// WhyDepth returns 0. +func WhyDepth(path string) int { + n := 0 + pkg, _ := loaded.pkgCache.Get(path) + for p := pkg; p != nil; p = p.stack { + n++ + } + return n +} diff --git a/src/cmd/go/internal/modload/modfile.go b/src/cmd/go/internal/modload/modfile.go new file mode 100644 index 0000000..899f1b3 --- /dev/null +++ b/src/cmd/go/internal/modload/modfile.go @@ -0,0 +1,813 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "unicode" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/gover" + "cmd/go/internal/lockedfile" + "cmd/go/internal/modfetch" + "cmd/go/internal/par" + "cmd/go/internal/trace" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" +) + +// ReadModFile reads and parses the mod file at gomod. ReadModFile properly applies the +// overlay, locks the file while reading, and applies fix, if applicable. +func ReadModFile(gomod string, fix modfile.VersionFixer) (data []byte, f *modfile.File, err error) { + gomod = base.ShortPath(gomod) // use short path in any errors + if gomodActual, ok := fsys.OverlayPath(gomod); ok { + // Don't lock go.mod if it's part of the overlay. + // On Plan 9, locking requires chmod, and we don't want to modify any file + // in the overlay. See #44700. + data, err = os.ReadFile(gomodActual) + } else { + data, err = lockedfile.Read(gomodActual) + } + if err != nil { + return nil, nil, err + } + + f, err = modfile.Parse(gomod, data, fix) + if err != nil { + // Errors returned by modfile.Parse begin with file:line. + return nil, nil, fmt.Errorf("errors parsing %s:\n%w", gomod, err) + } + if f.Go != nil && gover.Compare(f.Go.Version, gover.Local()) > 0 { + toolchain := "" + if f.Toolchain != nil { + toolchain = f.Toolchain.Name + } + return nil, nil, &gover.TooNewError{What: gomod, GoVersion: f.Go.Version, Toolchain: toolchain} + } + if f.Module == nil { + // No module declaration. Must add module path. + return nil, nil, fmt.Errorf("error reading %s: missing module declaration. To specify the module path:\n\tgo mod edit -module=example.com/mod", gomod) + } + + return data, f, err +} + +// A modFileIndex is an index of data corresponding to a modFile +// at a specific point in time. +type modFileIndex struct { + data []byte + dataNeedsFix bool // true if fixVersion applied a change while parsing data + module module.Version + goVersion string // Go version (no "v" or "go" prefix) + toolchain string + require map[module.Version]requireMeta + replace map[module.Version]module.Version + exclude map[module.Version]bool +} + +type requireMeta struct { + indirect bool +} + +// A modPruning indicates whether transitive dependencies of Go 1.17 dependencies +// are pruned out of the module subgraph rooted at a given module. +// (See https://golang.org/ref/mod#graph-pruning.) +type modPruning uint8 + +const ( + pruned modPruning = iota // transitive dependencies of modules at go 1.17 and higher are pruned out + unpruned // no transitive dependencies are pruned out + workspace // pruned to the union of modules in the workspace +) + +func (p modPruning) String() string { + switch p { + case pruned: + return "pruned" + case unpruned: + return "unpruned" + case workspace: + return "workspace" + default: + return fmt.Sprintf("%T(%d)", p, p) + } +} + +func pruningForGoVersion(goVersion string) modPruning { + if gover.Compare(goVersion, gover.ExplicitIndirectVersion) < 0 { + // The go.mod file does not duplicate relevant information about transitive + // dependencies, so they cannot be pruned out. + return unpruned + } + return pruned +} + +// CheckAllowed returns an error equivalent to ErrDisallowed if m is excluded by +// the main module's go.mod or retracted by its author. Most version queries use +// this to filter out versions that should not be used. +func CheckAllowed(ctx context.Context, m module.Version) error { + if err := CheckExclusions(ctx, m); err != nil { + return err + } + if err := CheckRetractions(ctx, m); err != nil { + return err + } + return nil +} + +// ErrDisallowed is returned by version predicates passed to Query and similar +// functions to indicate that a version should not be considered. +var ErrDisallowed = errors.New("disallowed module version") + +// CheckExclusions returns an error equivalent to ErrDisallowed if module m is +// excluded by the main module's go.mod file. +func CheckExclusions(ctx context.Context, m module.Version) error { + for _, mainModule := range MainModules.Versions() { + if index := MainModules.Index(mainModule); index != nil && index.exclude[m] { + return module.VersionError(m, errExcluded) + } + } + return nil +} + +var errExcluded = &excludedError{} + +type excludedError struct{} + +func (e *excludedError) Error() string { return "excluded by go.mod" } +func (e *excludedError) Is(err error) bool { return err == ErrDisallowed } + +// CheckRetractions returns an error if module m has been retracted by +// its author. +func CheckRetractions(ctx context.Context, m module.Version) (err error) { + defer func() { + if retractErr := (*ModuleRetractedError)(nil); err == nil || errors.As(err, &retractErr) { + return + } + // Attribute the error to the version being checked, not the version from + // which the retractions were to be loaded. + if mErr := (*module.ModuleError)(nil); errors.As(err, &mErr) { + err = mErr.Err + } + err = &retractionLoadingError{m: m, err: err} + }() + + if m.Version == "" { + // Main module, standard library, or file replacement module. + // Cannot be retracted. + return nil + } + if repl := Replacement(module.Version{Path: m.Path}); repl.Path != "" { + // All versions of the module were replaced. + // Don't load retractions, since we'd just load the replacement. + return nil + } + + // Find the latest available version of the module, and load its go.mod. If + // the latest version is replaced, we'll load the replacement. + // + // If there's an error loading the go.mod, we'll return it here. These errors + // should generally be ignored by callers since they happen frequently when + // we're offline. These errors are not equivalent to ErrDisallowed, so they + // may be distinguished from retraction errors. + // + // We load the raw file here: the go.mod file may have a different module + // path that we expect if the module or its repository was renamed. + // We still want to apply retractions to other aliases of the module. + rm, err := queryLatestVersionIgnoringRetractions(ctx, m.Path) + if err != nil { + return err + } + summary, err := rawGoModSummary(rm) + if err != nil { + return err + } + + var rationale []string + isRetracted := false + for _, r := range summary.retract { + if gover.ModCompare(m.Path, r.Low, m.Version) <= 0 && gover.ModCompare(m.Path, m.Version, r.High) <= 0 { + isRetracted = true + if r.Rationale != "" { + rationale = append(rationale, r.Rationale) + } + } + } + if isRetracted { + return module.VersionError(m, &ModuleRetractedError{Rationale: rationale}) + } + return nil +} + +type ModuleRetractedError struct { + Rationale []string +} + +func (e *ModuleRetractedError) Error() string { + msg := "retracted by module author" + if len(e.Rationale) > 0 { + // This is meant to be a short error printed on a terminal, so just + // print the first rationale. + msg += ": " + ShortMessage(e.Rationale[0], "retracted by module author") + } + return msg +} + +func (e *ModuleRetractedError) Is(err error) bool { + return err == ErrDisallowed +} + +type retractionLoadingError struct { + m module.Version + err error +} + +func (e *retractionLoadingError) Error() string { + return fmt.Sprintf("loading module retractions for %v: %v", e.m, e.err) +} + +func (e *retractionLoadingError) Unwrap() error { + return e.err +} + +// ShortMessage returns a string from go.mod (for example, a retraction +// rationale or deprecation message) that is safe to print in a terminal. +// +// If the given string is empty, ShortMessage returns the given default. If the +// given string is too long or contains non-printable characters, ShortMessage +// returns a hard-coded string. +func ShortMessage(message, emptyDefault string) string { + const maxLen = 500 + if i := strings.Index(message, "\n"); i >= 0 { + message = message[:i] + } + message = strings.TrimSpace(message) + if message == "" { + return emptyDefault + } + if len(message) > maxLen { + return "(message omitted: too long)" + } + for _, r := range message { + if !unicode.IsGraphic(r) && !unicode.IsSpace(r) { + return "(message omitted: contains non-printable characters)" + } + } + // NOTE: the go.mod parser rejects invalid UTF-8, so we don't check that here. + return message +} + +// CheckDeprecation returns a deprecation message from the go.mod file of the +// latest version of the given module. Deprecation messages are comments +// before or on the same line as the module directives that start with +// "Deprecated:" and run until the end of the paragraph. +// +// CheckDeprecation returns an error if the message can't be loaded. +// CheckDeprecation returns "", nil if there is no deprecation message. +func CheckDeprecation(ctx context.Context, m module.Version) (deprecation string, err error) { + defer func() { + if err != nil { + err = fmt.Errorf("loading deprecation for %s: %w", m.Path, err) + } + }() + + if m.Version == "" { + // Main module, standard library, or file replacement module. + // Don't look up deprecation. + return "", nil + } + if repl := Replacement(module.Version{Path: m.Path}); repl.Path != "" { + // All versions of the module were replaced. + // We'll look up deprecation separately for the replacement. + return "", nil + } + + latest, err := queryLatestVersionIgnoringRetractions(ctx, m.Path) + if err != nil { + return "", err + } + summary, err := rawGoModSummary(latest) + if err != nil { + return "", err + } + return summary.deprecated, nil +} + +func replacement(mod module.Version, replace map[module.Version]module.Version) (fromVersion string, to module.Version, ok bool) { + if r, ok := replace[mod]; ok { + return mod.Version, r, true + } + if r, ok := replace[module.Version{Path: mod.Path}]; ok { + return "", r, true + } + return "", module.Version{}, false +} + +// Replacement returns the replacement for mod, if any. If the path in the +// module.Version is relative it's relative to the single main module outside +// workspace mode, or the workspace's directory in workspace mode. +func Replacement(mod module.Version) module.Version { + r, foundModRoot, _ := replacementFrom(mod) + return canonicalizeReplacePath(r, foundModRoot) +} + +// replacementFrom returns the replacement for mod, if any, the modroot of the replacement if it appeared in a go.mod, +// and the source of the replacement. The replacement is relative to the go.work or go.mod file it appears in. +func replacementFrom(mod module.Version) (r module.Version, modroot string, fromFile string) { + foundFrom, found, foundModRoot := "", module.Version{}, "" + if MainModules == nil { + return module.Version{}, "", "" + } else if MainModules.Contains(mod.Path) && mod.Version == "" { + // Don't replace the workspace version of the main module. + return module.Version{}, "", "" + } + if _, r, ok := replacement(mod, MainModules.WorkFileReplaceMap()); ok { + return r, "", workFilePath + } + for _, v := range MainModules.Versions() { + if index := MainModules.Index(v); index != nil { + if from, r, ok := replacement(mod, index.replace); ok { + modRoot := MainModules.ModRoot(v) + if foundModRoot != "" && foundFrom != from && found != r { + base.Errorf("conflicting replacements found for %v in workspace modules defined by %v and %v", + mod, modFilePath(foundModRoot), modFilePath(modRoot)) + return found, foundModRoot, modFilePath(foundModRoot) + } + found, foundModRoot = r, modRoot + } + } + } + return found, foundModRoot, modFilePath(foundModRoot) +} + +func replaceRelativeTo() string { + if workFilePath := WorkFilePath(); workFilePath != "" { + return filepath.Dir(workFilePath) + } + return MainModules.ModRoot(MainModules.mustGetSingleMainModule()) +} + +// canonicalizeReplacePath ensures that relative, on-disk, replaced module paths +// are relative to the workspace directory (in workspace mode) or to the module's +// directory (in module mode, as they already are). +func canonicalizeReplacePath(r module.Version, modRoot string) module.Version { + if filepath.IsAbs(r.Path) || r.Version != "" || modRoot == "" { + return r + } + workFilePath := WorkFilePath() + if workFilePath == "" { + return r + } + abs := filepath.Join(modRoot, r.Path) + if rel, err := filepath.Rel(filepath.Dir(workFilePath), abs); err == nil { + return module.Version{Path: ToDirectoryPath(rel), Version: r.Version} + } + // We couldn't make the version's path relative to the workspace's path, + // so just return the absolute path. It's the best we can do. + return module.Version{Path: ToDirectoryPath(abs), Version: r.Version} +} + +// resolveReplacement returns the module actually used to load the source code +// for m: either m itself, or the replacement for m (iff m is replaced). +// It also returns the modroot of the module providing the replacement if +// one was found. +func resolveReplacement(m module.Version) module.Version { + if r := Replacement(m); r.Path != "" { + return r + } + return m +} + +func toReplaceMap(replacements []*modfile.Replace) map[module.Version]module.Version { + replaceMap := make(map[module.Version]module.Version, len(replacements)) + for _, r := range replacements { + if prev, dup := replaceMap[r.Old]; dup && prev != r.New { + base.Fatalf("go: conflicting replacements for %v:\n\t%v\n\t%v", r.Old, prev, r.New) + } + replaceMap[r.Old] = r.New + } + return replaceMap +} + +// indexModFile rebuilds the index of modFile. +// If modFile has been changed since it was first read, +// modFile.Cleanup must be called before indexModFile. +func indexModFile(data []byte, modFile *modfile.File, mod module.Version, needsFix bool) *modFileIndex { + i := new(modFileIndex) + i.data = data + i.dataNeedsFix = needsFix + + i.module = module.Version{} + if modFile.Module != nil { + i.module = modFile.Module.Mod + } + + i.goVersion = "" + if modFile.Go == nil { + rawGoVersion.Store(mod, "") + } else { + i.goVersion = modFile.Go.Version + rawGoVersion.Store(mod, modFile.Go.Version) + } + if modFile.Toolchain != nil { + i.toolchain = modFile.Toolchain.Name + } + + i.require = make(map[module.Version]requireMeta, len(modFile.Require)) + for _, r := range modFile.Require { + i.require[r.Mod] = requireMeta{indirect: r.Indirect} + } + + i.replace = toReplaceMap(modFile.Replace) + + i.exclude = make(map[module.Version]bool, len(modFile.Exclude)) + for _, x := range modFile.Exclude { + i.exclude[x.Mod] = true + } + + return i +} + +// modFileIsDirty reports whether the go.mod file differs meaningfully +// from what was indexed. +// If modFile has been changed (even cosmetically) since it was first read, +// modFile.Cleanup must be called before modFileIsDirty. +func (i *modFileIndex) modFileIsDirty(modFile *modfile.File) bool { + if i == nil { + return modFile != nil + } + + if i.dataNeedsFix { + return true + } + + if modFile.Module == nil { + if i.module != (module.Version{}) { + return true + } + } else if modFile.Module.Mod != i.module { + return true + } + + var goV, toolchain string + if modFile.Go != nil { + goV = modFile.Go.Version + } + if modFile.Toolchain != nil { + toolchain = modFile.Toolchain.Name + } + + if goV != i.goVersion || + toolchain != i.toolchain || + len(modFile.Require) != len(i.require) || + len(modFile.Replace) != len(i.replace) || + len(modFile.Exclude) != len(i.exclude) { + return true + } + + for _, r := range modFile.Require { + if meta, ok := i.require[r.Mod]; !ok { + return true + } else if r.Indirect != meta.indirect { + if cfg.BuildMod == "readonly" { + // The module's requirements are consistent; only the "// indirect" + // comments that are wrong. But those are only guaranteed to be accurate + // after a "go mod tidy" — it's a good idea to run those before + // committing a change, but it's certainly not mandatory. + } else { + return true + } + } + } + + for _, r := range modFile.Replace { + if r.New != i.replace[r.Old] { + return true + } + } + + for _, x := range modFile.Exclude { + if !i.exclude[x.Mod] { + return true + } + } + + return false +} + +// rawGoVersion records the Go version parsed from each module's go.mod file. +// +// If a module is replaced, the version of the replacement is keyed by the +// replacement module.Version, not the version being replaced. +var rawGoVersion sync.Map // map[module.Version]string + +// A modFileSummary is a summary of a go.mod file for which we do not need to +// retain complete information — for example, the go.mod file of a dependency +// module. +type modFileSummary struct { + module module.Version + goVersion string + toolchain string + pruning modPruning + require []module.Version + retract []retraction + deprecated string +} + +// A retraction consists of a retracted version interval and rationale. +// retraction is like modfile.Retract, but it doesn't point to the syntax tree. +type retraction struct { + modfile.VersionInterval + Rationale string +} + +// goModSummary returns a summary of the go.mod file for module m, +// taking into account any replacements for m, exclusions of its dependencies, +// and/or vendoring. +// +// m must be a version in the module graph, reachable from the Target module. +// In readonly mode, the go.sum file must contain an entry for m's go.mod file +// (or its replacement). goModSummary must not be called for the Target module +// itself, as its requirements may change. Use rawGoModSummary for other +// module versions. +// +// The caller must not modify the returned summary. +func goModSummary(m module.Version) (*modFileSummary, error) { + if m.Version == "" && !inWorkspaceMode() && MainModules.Contains(m.Path) { + panic("internal error: goModSummary called on a main module") + } + if gover.IsToolchain(m.Path) { + return rawGoModSummary(m) + } + + if cfg.BuildMod == "vendor" { + summary := &modFileSummary{ + module: module.Version{Path: m.Path}, + } + + readVendorList(VendorDir()) + if vendorVersion[m.Path] != m.Version { + // This module is not vendored, so packages cannot be loaded from it and + // it cannot be relevant to the build. + return summary, nil + } + + // For every module other than the target, + // return the full list of modules from modules.txt. + // We don't know what versions the vendored module actually relies on, + // so assume that it requires everything. + summary.require = vendorList + return summary, nil + } + + actual := resolveReplacement(m) + if mustHaveSums() && actual.Version != "" { + key := module.Version{Path: actual.Path, Version: actual.Version + "/go.mod"} + if !modfetch.HaveSum(key) { + suggestion := fmt.Sprintf(" for go.mod file; to add it:\n\tgo mod download %s", m.Path) + return nil, module.VersionError(actual, &sumMissingError{suggestion: suggestion}) + } + } + summary, err := rawGoModSummary(actual) + if err != nil { + return nil, err + } + + if actual.Version == "" { + // The actual module is a filesystem-local replacement, for which we have + // unfortunately not enforced any sort of invariants about module lines or + // matching module paths. Anything goes. + // + // TODO(bcmills): Remove this special-case, update tests, and add a + // release note. + } else { + if summary.module.Path == "" { + return nil, module.VersionError(actual, errors.New("parsing go.mod: missing module line")) + } + + // In theory we should only allow mpath to be unequal to m.Path here if the + // version that we fetched lacks an explicit go.mod file: if the go.mod file + // is explicit, then it should match exactly (to ensure that imports of other + // packages within the module are interpreted correctly). Unfortunately, we + // can't determine that information from the module proxy protocol: we'll have + // to leave that validation for when we load actual packages from within the + // module. + if mpath := summary.module.Path; mpath != m.Path && mpath != actual.Path { + return nil, module.VersionError(actual, + fmt.Errorf("parsing go.mod:\n"+ + "\tmodule declares its path as: %s\n"+ + "\t but was required as: %s", mpath, m.Path)) + } + } + + for _, mainModule := range MainModules.Versions() { + if index := MainModules.Index(mainModule); index != nil && len(index.exclude) > 0 { + // Drop any requirements on excluded versions. + // Don't modify the cached summary though, since we might need the raw + // summary separately. + haveExcludedReqs := false + for _, r := range summary.require { + if index.exclude[r] { + haveExcludedReqs = true + break + } + } + if haveExcludedReqs { + s := new(modFileSummary) + *s = *summary + s.require = make([]module.Version, 0, len(summary.require)) + for _, r := range summary.require { + if !index.exclude[r] { + s.require = append(s.require, r) + } + } + summary = s + } + } + } + return summary, nil +} + +// rawGoModSummary returns a new summary of the go.mod file for module m, +// ignoring all replacements that may apply to m and excludes that may apply to +// its dependencies. +// +// rawGoModSummary cannot be used on the main module outside of workspace mode. +func rawGoModSummary(m module.Version) (*modFileSummary, error) { + if gover.IsToolchain(m.Path) { + if m.Path == "go" && gover.Compare(m.Version, gover.GoStrictVersion) >= 0 { + // Declare that go 1.21.3 requires toolchain 1.21.3, + // so that go get knows that downgrading toolchain implies downgrading go + // and similarly upgrading go requires upgrading the toolchain. + return &modFileSummary{module: m, require: []module.Version{{Path: "toolchain", Version: "go" + m.Version}}}, nil + } + return &modFileSummary{module: m}, nil + } + if m.Version == "" && !inWorkspaceMode() && MainModules.Contains(m.Path) { + // Calling rawGoModSummary implies that we are treating m as a module whose + // requirements aren't the roots of the module graph and can't be modified. + // + // If we are not in workspace mode, then the requirements of the main module + // are the roots of the module graph and we expect them to be kept consistent. + panic("internal error: rawGoModSummary called on a main module") + } + if m.Version == "" && inWorkspaceMode() && m.Path == "command-line-arguments" { + // "go work sync" calls LoadModGraph to make sure the module graph is valid. + // If there are no modules in the workspace, we synthesize an empty + // command-line-arguments module, which rawGoModData cannot read a go.mod for. + return &modFileSummary{module: m}, nil + } + return rawGoModSummaryCache.Do(m, func() (*modFileSummary, error) { + summary := new(modFileSummary) + name, data, err := rawGoModData(m) + if err != nil { + return nil, err + } + f, err := modfile.ParseLax(name, data, nil) + if err != nil { + return nil, module.VersionError(m, fmt.Errorf("parsing %s: %v", base.ShortPath(name), err)) + } + if f.Module != nil { + summary.module = f.Module.Mod + summary.deprecated = f.Module.Deprecated + } + if f.Go != nil { + rawGoVersion.LoadOrStore(m, f.Go.Version) + summary.goVersion = f.Go.Version + summary.pruning = pruningForGoVersion(f.Go.Version) + } else { + summary.pruning = unpruned + } + if f.Toolchain != nil { + summary.toolchain = f.Toolchain.Name + } + if len(f.Require) > 0 { + summary.require = make([]module.Version, 0, len(f.Require)+1) + for _, req := range f.Require { + summary.require = append(summary.require, req.Mod) + } + } + if summary.goVersion != "" && gover.Compare(summary.goVersion, gover.GoStrictVersion) >= 0 { + if gover.Compare(summary.goVersion, gover.Local()) > 0 { + return nil, &gover.TooNewError{What: "module " + m.String(), GoVersion: summary.goVersion} + } + summary.require = append(summary.require, module.Version{Path: "go", Version: summary.goVersion}) + } + if len(f.Retract) > 0 { + summary.retract = make([]retraction, 0, len(f.Retract)) + for _, ret := range f.Retract { + summary.retract = append(summary.retract, retraction{ + VersionInterval: ret.VersionInterval, + Rationale: ret.Rationale, + }) + } + } + + return summary, nil + }) +} + +var rawGoModSummaryCache par.ErrCache[module.Version, *modFileSummary] + +// rawGoModData returns the content of the go.mod file for module m, ignoring +// all replacements that may apply to m. +// +// rawGoModData cannot be used on the main module outside of workspace mode. +// +// Unlike rawGoModSummary, rawGoModData does not cache its results in memory. +// Use rawGoModSummary instead unless you specifically need these bytes. +func rawGoModData(m module.Version) (name string, data []byte, err error) { + if m.Version == "" { + dir := m.Path + if !filepath.IsAbs(dir) { + if inWorkspaceMode() && MainModules.Contains(m.Path) { + dir = MainModules.ModRoot(m) + } else { + // m is a replacement module with only a file path. + dir = filepath.Join(replaceRelativeTo(), dir) + } + } + name = filepath.Join(dir, "go.mod") + if gomodActual, ok := fsys.OverlayPath(name); ok { + // Don't lock go.mod if it's part of the overlay. + // On Plan 9, locking requires chmod, and we don't want to modify any file + // in the overlay. See #44700. + data, err = os.ReadFile(gomodActual) + } else { + data, err = lockedfile.Read(gomodActual) + } + if err != nil { + return "", nil, module.VersionError(m, fmt.Errorf("reading %s: %v", base.ShortPath(name), err)) + } + } else { + if !gover.ModIsValid(m.Path, m.Version) { + // Disallow the broader queries supported by fetch.Lookup. + base.Fatalf("go: internal error: %s@%s: unexpected invalid semantic version", m.Path, m.Version) + } + name = "go.mod" + data, err = modfetch.GoMod(context.TODO(), m.Path, m.Version) + } + return name, data, err +} + +// queryLatestVersionIgnoringRetractions looks up the latest version of the +// module with the given path without considering retracted or excluded +// versions. +// +// If all versions of the module are replaced, +// queryLatestVersionIgnoringRetractions returns the replacement without making +// a query. +// +// If the queried latest version is replaced, +// queryLatestVersionIgnoringRetractions returns the replacement. +func queryLatestVersionIgnoringRetractions(ctx context.Context, path string) (latest module.Version, err error) { + return latestVersionIgnoringRetractionsCache.Do(path, func() (module.Version, error) { + ctx, span := trace.StartSpan(ctx, "queryLatestVersionIgnoringRetractions "+path) + defer span.Done() + + if repl := Replacement(module.Version{Path: path}); repl.Path != "" { + // All versions of the module were replaced. + // No need to query. + return repl, nil + } + + // Find the latest version of the module. + // Ignore exclusions from the main module's go.mod. + const ignoreSelected = "" + var allowAll AllowedFunc + rev, err := Query(ctx, path, "latest", ignoreSelected, allowAll) + if err != nil { + return module.Version{}, err + } + latest := module.Version{Path: path, Version: rev.Version} + if repl := resolveReplacement(latest); repl.Path != "" { + latest = repl + } + return latest, nil + }) +} + +var latestVersionIgnoringRetractionsCache par.ErrCache[string, module.Version] // path → queryLatestVersionIgnoringRetractions result + +// ToDirectoryPath adds a prefix if necessary so that path in unambiguously +// an absolute path or a relative path starting with a '.' or '..' +// path component. +func ToDirectoryPath(path string) string { + if modfile.IsDirectoryPath(path) { + return path + } + // The path is not a relative path or an absolute path, so make it relative + // to the current directory. + return "./" + filepath.ToSlash(filepath.Clean(path)) +} diff --git a/src/cmd/go/internal/modload/mvs.go b/src/cmd/go/internal/modload/mvs.go new file mode 100644 index 0000000..8ae2dbf --- /dev/null +++ b/src/cmd/go/internal/modload/mvs.go @@ -0,0 +1,136 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "context" + "errors" + "os" + "sort" + + "cmd/go/internal/gover" + "cmd/go/internal/modfetch" + "cmd/go/internal/modfetch/codehost" + + "golang.org/x/mod/module" +) + +// cmpVersion implements the comparison for versions in the module loader. +// +// It is consistent with gover.ModCompare except that as a special case, +// the version "" is considered higher than all other versions. +// The main module (also known as the target) has no version and must be chosen +// over other versions of the same module in the module dependency graph. +func cmpVersion(p string, v1, v2 string) int { + if v2 == "" { + if v1 == "" { + return 0 + } + return -1 + } + if v1 == "" { + return 1 + } + return gover.ModCompare(p, v1, v2) +} + +// mvsReqs implements mvs.Reqs for module semantic versions, +// with any exclusions or replacements applied internally. +type mvsReqs struct { + roots []module.Version +} + +func (r *mvsReqs) Required(mod module.Version) ([]module.Version, error) { + if mod.Version == "" && MainModules.Contains(mod.Path) { + // Use the build list as it existed when r was constructed, not the current + // global build list. + return r.roots, nil + } + + if mod.Version == "none" { + return nil, nil + } + + summary, err := goModSummary(mod) + if err != nil { + return nil, err + } + return summary.require, nil +} + +// Max returns the maximum of v1 and v2 according to gover.ModCompare. +// +// As a special case, the version "" is considered higher than all other +// versions. The main module (also known as the target) has no version and must +// be chosen over other versions of the same module in the module dependency +// graph. +func (*mvsReqs) Max(p, v1, v2 string) string { + if cmpVersion(p, v1, v2) < 0 { + return v2 + } + return v1 +} + +// Upgrade is a no-op, here to implement mvs.Reqs. +// The upgrade logic for go get -u is in ../modget/get.go. +func (*mvsReqs) Upgrade(m module.Version) (module.Version, error) { + return m, nil +} + +func versions(ctx context.Context, path string, allowed AllowedFunc) (versions []string, origin *codehost.Origin, err error) { + // Note: modfetch.Lookup and repo.Versions are cached, + // so there's no need for us to add extra caching here. + err = modfetch.TryProxies(func(proxy string) error { + repo, err := lookupRepo(ctx, proxy, path) + if err != nil { + return err + } + allVersions, err := repo.Versions(ctx, "") + if err != nil { + return err + } + allowedVersions := make([]string, 0, len(allVersions.List)) + for _, v := range allVersions.List { + if err := allowed(ctx, module.Version{Path: path, Version: v}); err == nil { + allowedVersions = append(allowedVersions, v) + } else if !errors.Is(err, ErrDisallowed) { + return err + } + } + versions = allowedVersions + origin = allVersions.Origin + return nil + }) + return versions, origin, err +} + +// previousVersion returns the tagged version of m.Path immediately prior to +// m.Version, or version "none" if no prior version is tagged. +// +// Since the version of a main module is not found in the version list, +// it has no previous version. +func previousVersion(ctx context.Context, m module.Version) (module.Version, error) { + if m.Version == "" && MainModules.Contains(m.Path) { + return module.Version{Path: m.Path, Version: "none"}, nil + } + + list, _, err := versions(ctx, m.Path, CheckAllowed) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return module.Version{Path: m.Path, Version: "none"}, nil + } + return module.Version{}, err + } + i := sort.Search(len(list), func(i int) bool { return gover.ModCompare(m.Path, list[i], m.Version) >= 0 }) + if i > 0 { + return module.Version{Path: m.Path, Version: list[i-1]}, nil + } + return module.Version{Path: m.Path, Version: "none"}, nil +} + +func (*mvsReqs) Previous(m module.Version) (module.Version, error) { + // TODO(golang.org/issue/38714): thread tracing context through MVS. + return previousVersion(context.TODO(), m) +} diff --git a/src/cmd/go/internal/modload/mvs_test.go b/src/cmd/go/internal/modload/mvs_test.go new file mode 100644 index 0000000..e0a38b9 --- /dev/null +++ b/src/cmd/go/internal/modload/mvs_test.go @@ -0,0 +1,31 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "testing" +) + +func TestReqsMax(t *testing.T) { + type testCase struct { + a, b, want string + } + reqs := new(mvsReqs) + for _, tc := range []testCase{ + {a: "v0.1.0", b: "v0.2.0", want: "v0.2.0"}, + {a: "v0.2.0", b: "v0.1.0", want: "v0.2.0"}, + {a: "", b: "v0.1.0", want: ""}, // "" is Target.Version + {a: "v0.1.0", b: "", want: ""}, + {a: "none", b: "v0.1.0", want: "v0.1.0"}, + {a: "v0.1.0", b: "none", want: "v0.1.0"}, + {a: "none", b: "", want: ""}, + {a: "", b: "none", want: ""}, + } { + max := reqs.Max("", tc.a, tc.b) + if max != tc.want { + t.Errorf("(%T).Max(%q, %q) = %q; want %q", reqs, tc.a, tc.b, max, tc.want) + } + } +} diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go new file mode 100644 index 0000000..c4cf554 --- /dev/null +++ b/src/cmd/go/internal/modload/query.go @@ -0,0 +1,1343 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "bytes" + "context" + "errors" + "fmt" + "io/fs" + "os" + pathpkg "path" + "slices" + "sort" + "strings" + "sync" + "time" + + "cmd/go/internal/cfg" + "cmd/go/internal/gover" + "cmd/go/internal/imports" + "cmd/go/internal/modfetch" + "cmd/go/internal/modfetch/codehost" + "cmd/go/internal/modinfo" + "cmd/go/internal/search" + "cmd/go/internal/str" + "cmd/go/internal/trace" + "cmd/internal/pkgpattern" + + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +// Query looks up a revision of a given module given a version query string. +// The module must be a complete module path. +// The version must take one of the following forms: +// +// - the literal string "latest", denoting the latest available, allowed +// tagged version, with non-prereleases preferred over prereleases. +// If there are no tagged versions in the repo, latest returns the most +// recent commit. +// +// - the literal string "upgrade", equivalent to "latest" except that if +// current is a newer version, current will be returned (see below). +// +// - the literal string "patch", denoting the latest available tagged version +// with the same major and minor number as current (see below). +// +// - v1, denoting the latest available tagged version v1.x.x. +// +// - v1.2, denoting the latest available tagged version v1.2.x. +// +// - v1.2.3, a semantic version string denoting that tagged version. +// +// - <v1.2.3, <=v1.2.3, >v1.2.3, >=v1.2.3, +// denoting the version closest to the target and satisfying the given operator, +// with non-prereleases preferred over prereleases. +// +// - a repository commit identifier or tag, denoting that commit. +// +// current denotes the currently-selected version of the module; it may be +// "none" if no version is currently selected, or "" if the currently-selected +// version is unknown or should not be considered. If query is +// "upgrade" or "patch", current will be returned if it is a newer +// semantic version or a chronologically later pseudo-version than the +// version that would otherwise be chosen. This prevents accidental downgrades +// from newer pre-release or development versions. +// +// The allowed function (which may be nil) is used to filter out unsuitable +// versions (see AllowedFunc documentation for details). If the query refers to +// a specific revision (for example, "master"; see IsRevisionQuery), and the +// revision is disallowed by allowed, Query returns the error. If the query +// does not refer to a specific revision (for example, "latest"), Query +// acts as if versions disallowed by allowed do not exist. +// +// If path is the path of the main module and the query is "latest", +// Query returns Target.Version as the version. +// +// Query often returns a non-nil *RevInfo with a non-nil error, +// to provide an info.Origin that can allow the error to be cached. +func Query(ctx context.Context, path, query, current string, allowed AllowedFunc) (*modfetch.RevInfo, error) { + ctx, span := trace.StartSpan(ctx, "modload.Query "+path) + defer span.Done() + + return queryReuse(ctx, path, query, current, allowed, nil) +} + +// queryReuse is like Query but also takes a map of module info that can be reused +// if the validation criteria in Origin are met. +func queryReuse(ctx context.Context, path, query, current string, allowed AllowedFunc, reuse map[module.Version]*modinfo.ModulePublic) (*modfetch.RevInfo, error) { + var info *modfetch.RevInfo + err := modfetch.TryProxies(func(proxy string) (err error) { + info, err = queryProxy(ctx, proxy, path, query, current, allowed, reuse) + return err + }) + return info, err +} + +// checkReuse checks whether a revision of a given module +// for a given module may be reused, according to the information in origin. +func checkReuse(ctx context.Context, m module.Version, old *codehost.Origin) error { + return modfetch.TryProxies(func(proxy string) error { + repo, err := lookupRepo(ctx, proxy, m.Path) + if err != nil { + return err + } + return checkReuseRepo(ctx, repo, m.Path, m.Version, old) + }) +} + +func checkReuseRepo(ctx context.Context, repo versionRepo, path, query string, origin *codehost.Origin) error { + if origin == nil { + return errors.New("nil Origin") + } + + // Ensure that the Origin actually includes enough fields to resolve the query. + // If we got the previous Origin data from a proxy, it may be missing something + // that we would have needed to resolve the query directly from the repo. + switch { + case origin.RepoSum != "": + // A RepoSum is always acceptable, since it incorporates everything + // (and is often associated with an error result). + + case query == module.CanonicalVersion(query): + // This query refers to a specific version, and Go module versions + // are supposed to be cacheable and immutable (confirmed with checksums). + // If the version exists at all, we shouldn't need any extra information + // to identify which commit it resolves to. + // + // It may be associated with a Ref for a semantic-version tag, but if so + // we don't expect that tag to change in the future. We also don't need a + // TagSum: if a tag is removed from some ancestor commit, the version may + // change from valid to invalid, but we're ok with keeping stale versions + // as long as they were valid at some point in the past. + // + // If the version did not successfully resolve, the origin may indicate + // a TagSum and/or RepoSum instead of a Hash, in which case we still need + // to check those to ensure that the error is still applicable. + if origin.Hash == "" && origin.Ref == "" && origin.TagSum == "" { + return errors.New("no Origin information to check") + } + + case IsRevisionQuery(path, query): + // This query may refer to a branch, non-version tag, or commit ID. + // + // If it is a commit ID, we expect to see a Hash in the Origin data. On + // the other hand, if it is not a commit ID, we expect to see either a Ref + // (for a positive result) or a RepoSum (for a negative result), since + // we don't expect refs in general to remain stable over time. + if origin.Hash == "" && origin.Ref == "" { + return fmt.Errorf("query %q requires a Hash or Ref", query) + } + // Once we resolve the query to a particular commit, we will need to + // also identify the most appropriate version to assign to that commit. + // (It may correspond to more than one valid version.) + // + // The most appropriate version depends on the tags associated with + // both the commit itself (if the commit is a tagged version) + // and its ancestors (if we need to produce a pseudo-version for it). + if origin.TagSum == "" { + return fmt.Errorf("query %q requires a TagSum", query) + } + + default: + // The query may be "latest" or a version inequality or prefix. + // Its result depends on the absence of higher tags matching the query, + // not just the state of an individual ref or tag. + if origin.TagSum == "" { + return fmt.Errorf("query %q requires a TagSum", query) + } + } + + return repo.CheckReuse(ctx, origin) +} + +// AllowedFunc is used by Query and other functions to filter out unsuitable +// versions, for example, those listed in exclude directives in the main +// module's go.mod file. +// +// An AllowedFunc returns an error equivalent to ErrDisallowed for an unsuitable +// version. Any other error indicates the function was unable to determine +// whether the version should be allowed, for example, the function was unable +// to fetch or parse a go.mod file containing retractions. Typically, errors +// other than ErrDisallowed may be ignored. +type AllowedFunc func(context.Context, module.Version) error + +var errQueryDisabled error = queryDisabledError{} + +type queryDisabledError struct{} + +func (queryDisabledError) Error() string { + if cfg.BuildModReason == "" { + return fmt.Sprintf("cannot query module due to -mod=%s", cfg.BuildMod) + } + return fmt.Sprintf("cannot query module due to -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) +} + +func queryProxy(ctx context.Context, proxy, path, query, current string, allowed AllowedFunc, reuse map[module.Version]*modinfo.ModulePublic) (*modfetch.RevInfo, error) { + ctx, span := trace.StartSpan(ctx, "modload.queryProxy "+path+" "+query) + defer span.Done() + + if current != "" && current != "none" && !gover.ModIsValid(path, current) { + return nil, fmt.Errorf("invalid previous version %v@%v", path, current) + } + if cfg.BuildMod == "vendor" { + return nil, errQueryDisabled + } + if allowed == nil { + allowed = func(context.Context, module.Version) error { return nil } + } + + if MainModules.Contains(path) && (query == "upgrade" || query == "patch") { + m := module.Version{Path: path} + if err := allowed(ctx, m); err != nil { + return nil, fmt.Errorf("internal error: main module version is not allowed: %w", err) + } + return &modfetch.RevInfo{Version: m.Version}, nil + } + + if path == "std" || path == "cmd" { + return nil, fmt.Errorf("can't query specific version (%q) of standard-library module %q", query, path) + } + + repo, err := lookupRepo(ctx, proxy, path) + if err != nil { + return nil, err + } + + if old := reuse[module.Version{Path: path, Version: query}]; old != nil { + if err := checkReuseRepo(ctx, repo, path, query, old.Origin); err == nil { + info := &modfetch.RevInfo{ + Version: old.Version, + Origin: old.Origin, + } + if old.Time != nil { + info.Time = *old.Time + } + return info, nil + } + } + + // Parse query to detect parse errors (and possibly handle query) + // before any network I/O. + qm, err := newQueryMatcher(path, query, current, allowed) + if (err == nil && qm.canStat) || err == errRevQuery { + // Direct lookup of a commit identifier or complete (non-prefix) semantic + // version. + + // If the identifier is not a canonical semver tag — including if it's a + // semver tag with a +metadata suffix — then modfetch.Stat will populate + // info.Version with a suitable pseudo-version. + info, err := repo.Stat(ctx, query) + if err != nil { + queryErr := err + // The full query doesn't correspond to a tag. If it is a semantic version + // with a +metadata suffix, see if there is a tag without that suffix: + // semantic versioning defines them to be equivalent. + canonicalQuery := module.CanonicalVersion(query) + if canonicalQuery != "" && query != canonicalQuery { + info, err = repo.Stat(ctx, canonicalQuery) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return info, err + } + } + if err != nil { + return info, queryErr + } + } + if err := allowed(ctx, module.Version{Path: path, Version: info.Version}); errors.Is(err, ErrDisallowed) { + return nil, err + } + return info, nil + } else if err != nil { + return nil, err + } + + // Load versions and execute query. + versions, err := repo.Versions(ctx, qm.prefix) + if err != nil { + return nil, err + } + origin := versions.Origin + + revWithOrigin := func(rev *modfetch.RevInfo) *modfetch.RevInfo { + if rev == nil { + if origin == nil { + return nil + } + return &modfetch.RevInfo{Origin: origin} + } + + clone := *rev + clone.Origin = origin + return &clone + } + + releases, prereleases, err := qm.filterVersions(ctx, versions.List) + if err != nil { + return revWithOrigin(nil), err + } + + lookup := func(v string) (*modfetch.RevInfo, error) { + rev, err := repo.Stat(ctx, v) + if rev != nil { + // Note that Stat can return a non-nil rev and a non-nil err, + // in order to provide origin information to make the error cacheable. + origin = mergeOrigin(origin, rev.Origin) + } + if err != nil { + return revWithOrigin(nil), err + } + + if (query == "upgrade" || query == "patch") && module.IsPseudoVersion(current) && !rev.Time.IsZero() { + // Don't allow "upgrade" or "patch" to move from a pseudo-version + // to a chronologically older version or pseudo-version. + // + // If the current version is a pseudo-version from an untagged branch, it + // may be semantically lower than the "latest" release or the latest + // pseudo-version on the main branch. A user on such a version is unlikely + // to intend to “upgrade” to a version that already existed at that point + // in time. + // + // We do this only if the current version is a pseudo-version: if the + // version is tagged, the author of the dependency module has given us + // explicit information about their intended precedence of this version + // relative to other versions, and we shouldn't contradict that + // information. (For example, v1.0.1 might be a backport of a fix already + // incorporated into v1.1.0, in which case v1.0.1 would be chronologically + // newer but v1.1.0 is still an “upgrade”; or v1.0.2 might be a revert of + // an unsuccessful fix in v1.0.1, in which case the v1.0.2 commit may be + // older than the v1.0.1 commit despite the tag itself being newer.) + currentTime, err := module.PseudoVersionTime(current) + if err == nil && rev.Time.Before(currentTime) { + if err := allowed(ctx, module.Version{Path: path, Version: current}); errors.Is(err, ErrDisallowed) { + return revWithOrigin(nil), err + } + rev, err = repo.Stat(ctx, current) + if rev != nil { + origin = mergeOrigin(origin, rev.Origin) + } + if err != nil { + return revWithOrigin(nil), err + } + return revWithOrigin(rev), nil + } + } + + return revWithOrigin(rev), nil + } + + if qm.preferLower { + if len(releases) > 0 { + return lookup(releases[0]) + } + if len(prereleases) > 0 { + return lookup(prereleases[0]) + } + } else { + if len(releases) > 0 { + return lookup(releases[len(releases)-1]) + } + if len(prereleases) > 0 { + return lookup(prereleases[len(prereleases)-1]) + } + } + + if qm.mayUseLatest { + latest, err := repo.Latest(ctx) + if latest != nil { + origin = mergeOrigin(origin, latest.Origin) + } + if err == nil { + if qm.allowsVersion(ctx, latest.Version) { + return lookup(latest.Version) + } + } else if !errors.Is(err, fs.ErrNotExist) { + return revWithOrigin(nil), err + } + } + + if (query == "upgrade" || query == "patch") && current != "" && current != "none" { + // "upgrade" and "patch" may stay on the current version if allowed. + if err := allowed(ctx, module.Version{Path: path, Version: current}); errors.Is(err, ErrDisallowed) { + return revWithOrigin(nil), err + } + return lookup(current) + } + + return revWithOrigin(nil), &NoMatchingVersionError{query: query, current: current} +} + +// IsRevisionQuery returns true if vers is a version query that may refer to +// a particular version or revision in a repository like "v1.0.0", "master", +// or "0123abcd". IsRevisionQuery returns false if vers is a query that +// chooses from among available versions like "latest" or ">v1.0.0". +func IsRevisionQuery(path, vers string) bool { + if vers == "latest" || + vers == "upgrade" || + vers == "patch" || + strings.HasPrefix(vers, "<") || + strings.HasPrefix(vers, ">") || + (gover.ModIsValid(path, vers) && gover.ModIsPrefix(path, vers)) { + return false + } + return true +} + +type queryMatcher struct { + path string + prefix string + filter func(version string) bool + allowed AllowedFunc + canStat bool // if true, the query can be resolved by repo.Stat + preferLower bool // if true, choose the lowest matching version + mayUseLatest bool + preferIncompatible bool +} + +var errRevQuery = errors.New("query refers to a non-semver revision") + +// newQueryMatcher returns a new queryMatcher that matches the versions +// specified by the given query on the module with the given path. +// +// If the query can only be resolved by statting a non-SemVer revision, +// newQueryMatcher returns errRevQuery. +func newQueryMatcher(path string, query, current string, allowed AllowedFunc) (*queryMatcher, error) { + badVersion := func(v string) (*queryMatcher, error) { + return nil, fmt.Errorf("invalid semantic version %q in range %q", v, query) + } + + matchesMajor := func(v string) bool { + _, pathMajor, ok := module.SplitPathVersion(path) + if !ok { + return false + } + return module.CheckPathMajor(v, pathMajor) == nil + } + + qm := &queryMatcher{ + path: path, + allowed: allowed, + preferIncompatible: strings.HasSuffix(current, "+incompatible"), + } + + switch { + case query == "latest": + qm.mayUseLatest = true + + case query == "upgrade": + if current == "" || current == "none" { + qm.mayUseLatest = true + } else { + qm.mayUseLatest = module.IsPseudoVersion(current) + qm.filter = func(mv string) bool { return gover.ModCompare(qm.path, mv, current) >= 0 } + } + + case query == "patch": + if current == "" || current == "none" { + return nil, &NoPatchBaseError{path} + } + if current == "" { + qm.mayUseLatest = true + } else { + qm.mayUseLatest = module.IsPseudoVersion(current) + qm.prefix = gover.ModMajorMinor(qm.path, current) + "." + qm.filter = func(mv string) bool { return gover.ModCompare(qm.path, mv, current) >= 0 } + } + + case strings.HasPrefix(query, "<="): + v := query[len("<="):] + if !gover.ModIsValid(path, v) { + return badVersion(v) + } + if gover.ModIsPrefix(path, v) { + // Refuse to say whether <=v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). + return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) + } + qm.filter = func(mv string) bool { return gover.ModCompare(qm.path, mv, v) <= 0 } + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case strings.HasPrefix(query, "<"): + v := query[len("<"):] + if !gover.ModIsValid(path, v) { + return badVersion(v) + } + qm.filter = func(mv string) bool { return gover.ModCompare(qm.path, mv, v) < 0 } + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case strings.HasPrefix(query, ">="): + v := query[len(">="):] + if !gover.ModIsValid(path, v) { + return badVersion(v) + } + qm.filter = func(mv string) bool { return gover.ModCompare(qm.path, mv, v) >= 0 } + qm.preferLower = true + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case strings.HasPrefix(query, ">"): + v := query[len(">"):] + if !gover.ModIsValid(path, v) { + return badVersion(v) + } + if gover.ModIsPrefix(path, v) { + // Refuse to say whether >v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). + return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) + } + qm.filter = func(mv string) bool { return gover.ModCompare(qm.path, mv, v) > 0 } + qm.preferLower = true + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case gover.ModIsValid(path, query): + if gover.ModIsPrefix(path, query) { + qm.prefix = query + "." + // Do not allow the query "v1.2" to match versions lower than "v1.2.0", + // such as prereleases for that version. (https://golang.org/issue/31972) + qm.filter = func(mv string) bool { return gover.ModCompare(qm.path, mv, query) >= 0 } + } else { + qm.canStat = true + qm.filter = func(mv string) bool { return gover.ModCompare(qm.path, mv, query) == 0 } + qm.prefix = semver.Canonical(query) + } + if !matchesMajor(query) { + qm.preferIncompatible = true + } + + default: + return nil, errRevQuery + } + + return qm, nil +} + +// allowsVersion reports whether version v is allowed by the prefix, filter, and +// AllowedFunc of qm. +func (qm *queryMatcher) allowsVersion(ctx context.Context, v string) bool { + if qm.prefix != "" && !strings.HasPrefix(v, qm.prefix) { + if gover.IsToolchain(qm.path) && strings.TrimSuffix(qm.prefix, ".") == v { + // Allow 1.21 to match "1.21." prefix. + } else { + return false + } + } + if qm.filter != nil && !qm.filter(v) { + return false + } + if qm.allowed != nil { + if err := qm.allowed(ctx, module.Version{Path: qm.path, Version: v}); errors.Is(err, ErrDisallowed) { + return false + } + } + return true +} + +// filterVersions classifies versions into releases and pre-releases, filtering +// out: +// 1. versions that do not satisfy the 'allowed' predicate, and +// 2. "+incompatible" versions, if a compatible one satisfies the predicate +// and the incompatible version is not preferred. +// +// If the allowed predicate returns an error not equivalent to ErrDisallowed, +// filterVersions returns that error. +func (qm *queryMatcher) filterVersions(ctx context.Context, versions []string) (releases, prereleases []string, err error) { + needIncompatible := qm.preferIncompatible + + var lastCompatible string + for _, v := range versions { + if !qm.allowsVersion(ctx, v) { + continue + } + + if !needIncompatible { + // We're not yet sure whether we need to include +incompatible versions. + // Keep track of the last compatible version we've seen, and use the + // presence (or absence) of a go.mod file in that version to decide: a + // go.mod file implies that the module author is supporting modules at a + // compatible version (and we should ignore +incompatible versions unless + // requested explicitly), while a lack of go.mod file implies the + // potential for legacy (pre-modules) versioning without semantic import + // paths (and thus *with* +incompatible versions). + // + // This isn't strictly accurate if the latest compatible version has been + // replaced by a local file path, because we do not allow file-path + // replacements without a go.mod file: the user would have needed to add + // one. However, replacing the last compatible version while + // simultaneously expecting to upgrade implicitly to a +incompatible + // version seems like an extreme enough corner case to ignore for now. + + if !strings.HasSuffix(v, "+incompatible") { + lastCompatible = v + } else if lastCompatible != "" { + // If the latest compatible version is allowed and has a go.mod file, + // ignore any version with a higher (+incompatible) major version. (See + // https://golang.org/issue/34165.) Note that we even prefer a + // compatible pre-release over an incompatible release. + ok, err := versionHasGoMod(ctx, module.Version{Path: qm.path, Version: lastCompatible}) + if err != nil { + return nil, nil, err + } + if ok { + // The last compatible version has a go.mod file, so that's the + // highest version we're willing to consider. Don't bother even + // looking at higher versions, because they're all +incompatible from + // here onward. + break + } + + // No acceptable compatible release has a go.mod file, so the versioning + // for the module might not be module-aware, and we should respect + // legacy major-version tags. + needIncompatible = true + } + } + + if gover.ModIsPrerelease(qm.path, v) { + prereleases = append(prereleases, v) + } else { + releases = append(releases, v) + } + } + + return releases, prereleases, nil +} + +type QueryResult struct { + Mod module.Version + Rev *modfetch.RevInfo + Packages []string +} + +// QueryPackages is like QueryPattern, but requires that the pattern match at +// least one package and omits the non-package result (if any). +func QueryPackages(ctx context.Context, pattern, query string, current func(string) string, allowed AllowedFunc) ([]QueryResult, error) { + pkgMods, modOnly, err := QueryPattern(ctx, pattern, query, current, allowed) + + if len(pkgMods) == 0 && err == nil { + replacement := Replacement(modOnly.Mod) + return nil, &PackageNotInModuleError{ + Mod: modOnly.Mod, + Replacement: replacement, + Query: query, + Pattern: pattern, + } + } + + return pkgMods, err +} + +// QueryPattern looks up the module(s) containing at least one package matching +// the given pattern at the given version. The results are sorted by module path +// length in descending order. If any proxy provides a non-empty set of candidate +// modules, no further proxies are tried. +// +// For wildcard patterns, QueryPattern looks in modules with package paths up to +// the first "..." in the pattern. For the pattern "example.com/a/b.../c", +// QueryPattern would consider prefixes of "example.com/a". +// +// If any matching package is in the main module, QueryPattern considers only +// the main module and only the version "latest", without checking for other +// possible modules. +// +// QueryPattern always returns at least one QueryResult (which may be only +// modOnly) or a non-nil error. +func QueryPattern(ctx context.Context, pattern, query string, current func(string) string, allowed AllowedFunc) (pkgMods []QueryResult, modOnly *QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.QueryPattern "+pattern+" "+query) + defer span.Done() + + base := pattern + + firstError := func(m *search.Match) error { + if len(m.Errs) == 0 { + return nil + } + return m.Errs[0] + } + + var match func(mod module.Version, roots []string, isLocal bool) *search.Match + matchPattern := pkgpattern.MatchPattern(pattern) + + if i := strings.Index(pattern, "..."); i >= 0 { + base = pathpkg.Dir(pattern[:i+3]) + if base == "." { + return nil, nil, &WildcardInFirstElementError{Pattern: pattern, Query: query} + } + match = func(mod module.Version, roots []string, isLocal bool) *search.Match { + m := search.NewMatch(pattern) + matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{mod}) + return m + } + } else { + match = func(mod module.Version, roots []string, isLocal bool) *search.Match { + m := search.NewMatch(pattern) + prefix := mod.Path + if MainModules.Contains(mod.Path) { + prefix = MainModules.PathPrefix(module.Version{Path: mod.Path}) + } + for _, root := range roots { + if _, ok, err := dirInModule(pattern, prefix, root, isLocal); err != nil { + m.AddError(err) + } else if ok { + m.Pkgs = []string{pattern} + } + } + return m + } + } + + var mainModuleMatches []module.Version + for _, mainModule := range MainModules.Versions() { + m := match(mainModule, modRoots, true) + if len(m.Pkgs) > 0 { + if query != "upgrade" && query != "patch" { + return nil, nil, &QueryMatchesPackagesInMainModuleError{ + Pattern: pattern, + Query: query, + Packages: m.Pkgs, + } + } + if err := allowed(ctx, mainModule); err != nil { + return nil, nil, fmt.Errorf("internal error: package %s is in the main module (%s), but version is not allowed: %w", pattern, mainModule.Path, err) + } + return []QueryResult{{ + Mod: mainModule, + Rev: &modfetch.RevInfo{Version: mainModule.Version}, + Packages: m.Pkgs, + }}, nil, nil + } + if err := firstError(m); err != nil { + return nil, nil, err + } + + var matchesMainModule bool + if matchPattern(mainModule.Path) { + mainModuleMatches = append(mainModuleMatches, mainModule) + matchesMainModule = true + } + + if (query == "upgrade" || query == "patch") && matchesMainModule { + if err := allowed(ctx, mainModule); err == nil { + modOnly = &QueryResult{ + Mod: mainModule, + Rev: &modfetch.RevInfo{Version: mainModule.Version}, + } + } + } + } + + var ( + results []QueryResult + candidateModules = modulePrefixesExcludingTarget(base) + ) + if len(candidateModules) == 0 { + if modOnly != nil { + return nil, modOnly, nil + } else if len(mainModuleMatches) != 0 { + return nil, nil, &QueryMatchesMainModulesError{ + MainModules: mainModuleMatches, + Pattern: pattern, + Query: query, + } + } else { + return nil, nil, &PackageNotInModuleError{ + MainModules: mainModuleMatches, + Query: query, + Pattern: pattern, + } + } + } + + err = modfetch.TryProxies(func(proxy string) error { + queryModule := func(ctx context.Context, path string) (r QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.QueryPattern.queryModule ["+proxy+"] "+path) + defer span.Done() + + pathCurrent := current(path) + r.Mod.Path = path + r.Rev, err = queryProxy(ctx, proxy, path, query, pathCurrent, allowed, nil) + if err != nil { + return r, err + } + r.Mod.Version = r.Rev.Version + if gover.IsToolchain(r.Mod.Path) { + return r, nil + } + root, isLocal, err := fetch(ctx, r.Mod) + if err != nil { + return r, err + } + m := match(r.Mod, []string{root}, isLocal) + r.Packages = m.Pkgs + if len(r.Packages) == 0 && !matchPattern(path) { + if err := firstError(m); err != nil { + return r, err + } + replacement := Replacement(r.Mod) + return r, &PackageNotInModuleError{ + Mod: r.Mod, + Replacement: replacement, + Query: query, + Pattern: pattern, + } + } + return r, nil + } + + allResults, err := queryPrefixModules(ctx, candidateModules, queryModule) + results = allResults[:0] + for _, r := range allResults { + if len(r.Packages) == 0 { + modOnly = &r + } else { + results = append(results, r) + } + } + return err + }) + + if len(mainModuleMatches) > 0 && len(results) == 0 && modOnly == nil && errors.Is(err, fs.ErrNotExist) { + return nil, nil, &QueryMatchesMainModulesError{ + Pattern: pattern, + Query: query, + } + } + return slices.Clip(results), modOnly, err +} + +// modulePrefixesExcludingTarget returns all prefixes of path that may plausibly +// exist as a module, excluding targetPrefix but otherwise including path +// itself, sorted by descending length. Prefixes that are not valid module paths +// but are valid package paths (like "m" or "example.com/.gen") are included, +// since they might be replaced. +func modulePrefixesExcludingTarget(path string) []string { + prefixes := make([]string, 0, strings.Count(path, "/")+1) + + mainModulePrefixes := make(map[string]bool) + for _, m := range MainModules.Versions() { + mainModulePrefixes[m.Path] = true + } + + for { + if !mainModulePrefixes[path] { + if _, _, ok := module.SplitPathVersion(path); ok { + prefixes = append(prefixes, path) + } + } + + j := strings.LastIndexByte(path, '/') + if j < 0 { + break + } + path = path[:j] + } + + return prefixes +} + +func queryPrefixModules(ctx context.Context, candidateModules []string, queryModule func(ctx context.Context, path string) (QueryResult, error)) (found []QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.queryPrefixModules") + defer span.Done() + + // If the path we're attempting is not in the module cache and we don't have a + // fetch result cached either, we'll end up making a (potentially slow) + // request to the proxy or (often even slower) the origin server. + // To minimize latency, execute all of those requests in parallel. + type result struct { + QueryResult + err error + } + results := make([]result, len(candidateModules)) + var wg sync.WaitGroup + wg.Add(len(candidateModules)) + for i, p := range candidateModules { + ctx := trace.StartGoroutine(ctx) + go func(p string, r *result) { + r.QueryResult, r.err = queryModule(ctx, p) + wg.Done() + }(p, &results[i]) + } + wg.Wait() + + // Classify the results. In case of failure, identify the error that the user + // is most likely to find helpful: the most useful class of error at the + // longest matching path. + var ( + noPackage *PackageNotInModuleError + noVersion *NoMatchingVersionError + noPatchBase *NoPatchBaseError + invalidPath *module.InvalidPathError // see comment in case below + invalidVersion error + notExistErr error + ) + for _, r := range results { + switch rErr := r.err.(type) { + case nil: + found = append(found, r.QueryResult) + case *PackageNotInModuleError: + // Given the option, prefer to attribute “package not in module” + // to modules other than the main one. + if noPackage == nil || MainModules.Contains(noPackage.Mod.Path) { + noPackage = rErr + } + case *NoMatchingVersionError: + if noVersion == nil { + noVersion = rErr + } + case *NoPatchBaseError: + if noPatchBase == nil { + noPatchBase = rErr + } + case *module.InvalidPathError: + // The prefix was not a valid module path, and there was no replacement. + // Prefixes like this may appear in candidateModules, since we handle + // replaced modules that weren't required in the repo lookup process + // (see lookupRepo). + // + // A shorter prefix may be a valid module path and may contain a valid + // import path, so this is a low-priority error. + if invalidPath == nil { + invalidPath = rErr + } + default: + if errors.Is(rErr, fs.ErrNotExist) { + if notExistErr == nil { + notExistErr = rErr + } + } else if iv := (*module.InvalidVersionError)(nil); errors.As(rErr, &iv) { + if invalidVersion == nil { + invalidVersion = rErr + } + } else if err == nil { + if len(found) > 0 || noPackage != nil { + // golang.org/issue/34094: If we have already found a module that + // could potentially contain the target package, ignore unclassified + // errors for modules with shorter paths. + + // golang.org/issue/34383 is a special case of this: if we have + // already found example.com/foo/v2@v2.0.0 with a matching go.mod + // file, ignore the error from example.com/foo@v2.0.0. + } else { + err = r.err + } + } + } + } + + // TODO(#26232): If len(found) == 0 and some of the errors are 4xx HTTP + // codes, have the auth package recheck the failed paths. + // If we obtain new credentials for any of them, re-run the above loop. + + if len(found) == 0 && err == nil { + switch { + case noPackage != nil: + err = noPackage + case noVersion != nil: + err = noVersion + case noPatchBase != nil: + err = noPatchBase + case invalidPath != nil: + err = invalidPath + case invalidVersion != nil: + err = invalidVersion + case notExistErr != nil: + err = notExistErr + default: + panic("queryPrefixModules: no modules found, but no error detected") + } + } + + return found, err +} + +// A NoMatchingVersionError indicates that Query found a module at the requested +// path, but not at any versions satisfying the query string and allow-function. +// +// NOTE: NoMatchingVersionError MUST NOT implement Is(fs.ErrNotExist). +// +// If the module came from a proxy, that proxy had to return a successful status +// code for the versions it knows about, and thus did not have the opportunity +// to return a non-400 status code to suppress fallback. +type NoMatchingVersionError struct { + query, current string +} + +func (e *NoMatchingVersionError) Error() string { + currentSuffix := "" + if (e.query == "upgrade" || e.query == "patch") && e.current != "" && e.current != "none" { + currentSuffix = fmt.Sprintf(" (current version is %s)", e.current) + } + return fmt.Sprintf("no matching versions for query %q", e.query) + currentSuffix +} + +// A NoPatchBaseError indicates that Query was called with the query "patch" +// but with a current version of "" or "none". +type NoPatchBaseError struct { + path string +} + +func (e *NoPatchBaseError) Error() string { + return fmt.Sprintf(`can't query version "patch" of module %s: no existing version is required`, e.path) +} + +// A WildcardInFirstElementError indicates that a pattern passed to QueryPattern +// had a wildcard in its first path element, and therefore had no pattern-prefix +// modules to search in. +type WildcardInFirstElementError struct { + Pattern string + Query string +} + +func (e *WildcardInFirstElementError) Error() string { + return fmt.Sprintf("no modules to query for %s@%s because first path element contains a wildcard", e.Pattern, e.Query) +} + +// A PackageNotInModuleError indicates that QueryPattern found a candidate +// module at the requested version, but that module did not contain any packages +// matching the requested pattern. +// +// NOTE: PackageNotInModuleError MUST NOT implement Is(fs.ErrNotExist). +// +// If the module came from a proxy, that proxy had to return a successful status +// code for the versions it knows about, and thus did not have the opportunity +// to return a non-400 status code to suppress fallback. +type PackageNotInModuleError struct { + MainModules []module.Version + Mod module.Version + Replacement module.Version + Query string + Pattern string +} + +func (e *PackageNotInModuleError) Error() string { + if len(e.MainModules) > 0 { + prefix := "workspace modules do" + if len(e.MainModules) == 1 { + prefix = fmt.Sprintf("main module (%s) does", e.MainModules[0]) + } + if strings.Contains(e.Pattern, "...") { + return fmt.Sprintf("%s not contain packages matching %s", prefix, e.Pattern) + } + return fmt.Sprintf("%s not contain package %s", prefix, e.Pattern) + } + + found := "" + if r := e.Replacement; r.Path != "" { + replacement := r.Path + if r.Version != "" { + replacement = fmt.Sprintf("%s@%s", r.Path, r.Version) + } + if e.Query == e.Mod.Version { + found = fmt.Sprintf(" (replaced by %s)", replacement) + } else { + found = fmt.Sprintf(" (%s, replaced by %s)", e.Mod.Version, replacement) + } + } else if e.Query != e.Mod.Version { + found = fmt.Sprintf(" (%s)", e.Mod.Version) + } + + if strings.Contains(e.Pattern, "...") { + return fmt.Sprintf("module %s@%s found%s, but does not contain packages matching %s", e.Mod.Path, e.Query, found, e.Pattern) + } + return fmt.Sprintf("module %s@%s found%s, but does not contain package %s", e.Mod.Path, e.Query, found, e.Pattern) +} + +func (e *PackageNotInModuleError) ImportPath() string { + if !strings.Contains(e.Pattern, "...") { + return e.Pattern + } + return "" +} + +// versionHasGoMod returns whether a version has a go.mod file. +// +// versionHasGoMod fetches the go.mod file (possibly a fake) and true if it +// contains anything other than a module directive with the same path. When a +// module does not have a real go.mod file, the go command acts as if it had one +// that only contained a module directive. Normal go.mod files created after +// 1.12 at least have a go directive. +// +// This function is a heuristic, since it's possible to commit a file that would +// pass this test. However, we only need a heuristic for determining whether +// +incompatible versions may be "latest", which is what this function is used +// for. +// +// This heuristic is useful for two reasons: first, when using a proxy, +// this lets us fetch from the .mod endpoint which is much faster than the .zip +// endpoint. The .mod file is used anyway, even if the .zip file contains a +// go.mod with different content. Second, if we don't fetch the .zip, then +// we don't need to verify it in go.sum. This makes 'go list -m -u' faster +// and simpler. +func versionHasGoMod(_ context.Context, m module.Version) (bool, error) { + _, data, err := rawGoModData(m) + if err != nil { + return false, err + } + isFake := bytes.Equal(data, modfetch.LegacyGoMod(m.Path)) + return !isFake, nil +} + +// A versionRepo is a subset of modfetch.Repo that can report information about +// available versions, but cannot fetch specific source files. +type versionRepo interface { + ModulePath() string + CheckReuse(context.Context, *codehost.Origin) error + Versions(ctx context.Context, prefix string) (*modfetch.Versions, error) + Stat(ctx context.Context, rev string) (*modfetch.RevInfo, error) + Latest(context.Context) (*modfetch.RevInfo, error) +} + +var _ versionRepo = modfetch.Repo(nil) + +func lookupRepo(ctx context.Context, proxy, path string) (repo versionRepo, err error) { + if path != "go" && path != "toolchain" { + err = module.CheckPath(path) + } + if err == nil { + repo = modfetch.Lookup(ctx, proxy, path) + } else { + repo = emptyRepo{path: path, err: err} + } + + if MainModules == nil { + return repo, err + } else if _, ok := MainModules.HighestReplaced()[path]; ok { + return &replacementRepo{repo: repo}, nil + } + + return repo, err +} + +// An emptyRepo is a versionRepo that contains no versions. +type emptyRepo struct { + path string + err error +} + +var _ versionRepo = emptyRepo{} + +func (er emptyRepo) ModulePath() string { return er.path } +func (er emptyRepo) CheckReuse(ctx context.Context, old *codehost.Origin) error { + return fmt.Errorf("empty repo") +} +func (er emptyRepo) Versions(ctx context.Context, prefix string) (*modfetch.Versions, error) { + return &modfetch.Versions{}, nil +} +func (er emptyRepo) Stat(ctx context.Context, rev string) (*modfetch.RevInfo, error) { + return nil, er.err +} +func (er emptyRepo) Latest(ctx context.Context) (*modfetch.RevInfo, error) { return nil, er.err } + +// A replacementRepo augments a versionRepo to include the replacement versions +// (if any) found in the main module's go.mod file. +// +// A replacementRepo suppresses "not found" errors for otherwise-nonexistent +// modules, so a replacementRepo should only be constructed for a module that +// actually has one or more valid replacements. +type replacementRepo struct { + repo versionRepo +} + +var _ versionRepo = (*replacementRepo)(nil) + +func (rr *replacementRepo) ModulePath() string { return rr.repo.ModulePath() } + +func (rr *replacementRepo) CheckReuse(ctx context.Context, old *codehost.Origin) error { + return fmt.Errorf("replacement repo") +} + +// Versions returns the versions from rr.repo augmented with any matching +// replacement versions. +func (rr *replacementRepo) Versions(ctx context.Context, prefix string) (*modfetch.Versions, error) { + repoVersions, err := rr.repo.Versions(ctx, prefix) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + repoVersions = new(modfetch.Versions) + } + + versions := repoVersions.List + for _, mm := range MainModules.Versions() { + if index := MainModules.Index(mm); index != nil && len(index.replace) > 0 { + path := rr.ModulePath() + for m := range index.replace { + if m.Path == path && strings.HasPrefix(m.Version, prefix) && m.Version != "" && !module.IsPseudoVersion(m.Version) { + versions = append(versions, m.Version) + } + } + } + } + + if len(versions) == len(repoVersions.List) { // replacement versions added + return repoVersions, nil + } + + path := rr.ModulePath() + sort.Slice(versions, func(i, j int) bool { + return gover.ModCompare(path, versions[i], versions[j]) < 0 + }) + str.Uniq(&versions) + return &modfetch.Versions{List: versions}, nil +} + +func (rr *replacementRepo) Stat(ctx context.Context, rev string) (*modfetch.RevInfo, error) { + info, err := rr.repo.Stat(ctx, rev) + if err == nil { + return info, err + } + var hasReplacements bool + for _, v := range MainModules.Versions() { + if index := MainModules.Index(v); index != nil && len(index.replace) > 0 { + hasReplacements = true + } + } + if !hasReplacements { + return info, err + } + + v := module.CanonicalVersion(rev) + if v != rev { + // The replacements in the go.mod file list only canonical semantic versions, + // so a non-canonical version can't possibly have a replacement. + return info, err + } + + path := rr.ModulePath() + _, pathMajor, ok := module.SplitPathVersion(path) + if ok && pathMajor == "" { + if err := module.CheckPathMajor(v, pathMajor); err != nil && semver.Build(v) == "" { + v += "+incompatible" + } + } + + if r := Replacement(module.Version{Path: path, Version: v}); r.Path == "" { + return info, err + } + return rr.replacementStat(v) +} + +func (rr *replacementRepo) Latest(ctx context.Context) (*modfetch.RevInfo, error) { + info, err := rr.repo.Latest(ctx) + path := rr.ModulePath() + + if v, ok := MainModules.HighestReplaced()[path]; ok { + if v == "" { + // The only replacement is a wildcard that doesn't specify a version, so + // synthesize a pseudo-version with an appropriate major version and a + // timestamp below any real timestamp. That way, if the main module is + // used from within some other module, the user will be able to upgrade + // the requirement to any real version they choose. + if _, pathMajor, ok := module.SplitPathVersion(path); ok && len(pathMajor) > 0 { + v = module.PseudoVersion(pathMajor[1:], "", time.Time{}, "000000000000") + } else { + v = module.PseudoVersion("v0", "", time.Time{}, "000000000000") + } + } + + if err != nil || gover.ModCompare(path, v, info.Version) > 0 { + return rr.replacementStat(v) + } + } + + return info, err +} + +func (rr *replacementRepo) replacementStat(v string) (*modfetch.RevInfo, error) { + rev := &modfetch.RevInfo{Version: v} + if module.IsPseudoVersion(v) { + rev.Time, _ = module.PseudoVersionTime(v) + rev.Short, _ = module.PseudoVersionRev(v) + } + return rev, nil +} + +// A QueryMatchesMainModulesError indicates that a query requests +// a version of the main module that cannot be satisfied. +// (The main module's version cannot be changed.) +type QueryMatchesMainModulesError struct { + MainModules []module.Version + Pattern string + Query string +} + +func (e *QueryMatchesMainModulesError) Error() string { + if MainModules.Contains(e.Pattern) { + return fmt.Sprintf("can't request version %q of the main module (%s)", e.Query, e.Pattern) + } + + plural := "" + mainModulePaths := make([]string, len(e.MainModules)) + for i := range e.MainModules { + mainModulePaths[i] = e.MainModules[i].Path + } + if len(e.MainModules) > 1 { + plural = "s" + } + return fmt.Sprintf("can't request version %q of pattern %q that includes the main module%s (%s)", e.Query, e.Pattern, plural, strings.Join(mainModulePaths, ", ")) +} + +// A QueryUpgradesAllError indicates that a query requests +// an upgrade on the all pattern. +// (The main module's version cannot be changed.) +type QueryUpgradesAllError struct { + MainModules []module.Version + Query string +} + +func (e *QueryUpgradesAllError) Error() string { + var plural string = "" + if len(e.MainModules) != 1 { + plural = "s" + } + + return fmt.Sprintf("can't request version %q of pattern \"all\" that includes the main module%s", e.Query, plural) +} + +// A QueryMatchesPackagesInMainModuleError indicates that a query cannot be +// satisfied because it matches one or more packages found in the main module. +type QueryMatchesPackagesInMainModuleError struct { + Pattern string + Query string + Packages []string +} + +func (e *QueryMatchesPackagesInMainModuleError) Error() string { + if len(e.Packages) > 1 { + return fmt.Sprintf("pattern %s matches %d packages in the main module, so can't request version %s", e.Pattern, len(e.Packages), e.Query) + } + + if search.IsMetaPackage(e.Pattern) || strings.Contains(e.Pattern, "...") { + return fmt.Sprintf("pattern %s matches package %s in the main module, so can't request version %s", e.Pattern, e.Packages[0], e.Query) + } + + return fmt.Sprintf("package %s is in the main module, so can't request version %s", e.Packages[0], e.Query) +} diff --git a/src/cmd/go/internal/modload/query_test.go b/src/cmd/go/internal/modload/query_test.go new file mode 100644 index 0000000..93f8f0d --- /dev/null +++ b/src/cmd/go/internal/modload/query_test.go @@ -0,0 +1,202 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "context" + "flag" + "internal/testenv" + "log" + "os" + "path" + "path/filepath" + "strings" + "testing" + + "cmd/go/internal/cfg" + "cmd/go/internal/vcweb/vcstest" + + "golang.org/x/mod/module" +) + +func TestMain(m *testing.M) { + flag.Parse() + if err := testMain(m); err != nil { + log.Fatal(err) + } +} + +func testMain(m *testing.M) (err error) { + cfg.GOPROXY = "direct" + cfg.ModCacheRW = true + + srv, err := vcstest.NewServer() + if err != nil { + return err + } + defer func() { + if closeErr := srv.Close(); err == nil { + err = closeErr + } + }() + + dir, err := os.MkdirTemp("", "modload-test-") + if err != nil { + return err + } + defer func() { + if rmErr := os.RemoveAll(dir); err == nil { + err = rmErr + } + }() + + os.Setenv("GOPATH", dir) + cfg.BuildContext.GOPATH = dir + cfg.GOMODCACHE = filepath.Join(dir, "pkg/mod") + cfg.SumdbDir = filepath.Join(dir, "pkg/sumdb") + m.Run() + return nil +} + +var ( + queryRepo = "vcs-test.golang.org/git/querytest.git" + queryRepoV2 = queryRepo + "/v2" + queryRepoV3 = queryRepo + "/v3" + + // Empty version list (no semver tags), not actually empty. + emptyRepoPath = "vcs-test.golang.org/git/emptytest.git" +) + +var queryTests = []struct { + path string + query string + current string + allow string + vers string + err string +}{ + {path: queryRepo, query: "<v0.0.0", vers: "v0.0.0-pre1"}, + {path: queryRepo, query: "<v0.0.0-pre1", err: `no matching versions for query "<v0.0.0-pre1"`}, + {path: queryRepo, query: "<=v0.0.0", vers: "v0.0.0"}, + {path: queryRepo, query: ">v0.0.0", vers: "v0.0.1"}, + {path: queryRepo, query: ">=v0.0.0", vers: "v0.0.0"}, + {path: queryRepo, query: "v0.0.1", vers: "v0.0.1"}, + {path: queryRepo, query: "v0.0.1+foo", vers: "v0.0.1"}, + {path: queryRepo, query: "v0.0.99", err: `vcs-test.golang.org/git/querytest.git@v0.0.99: invalid version: unknown revision v0.0.99`}, + {path: queryRepo, query: "v0", vers: "v0.3.0"}, + {path: queryRepo, query: "v0.1", vers: "v0.1.2"}, + {path: queryRepo, query: "v0.2", err: `no matching versions for query "v0.2"`}, + {path: queryRepo, query: "v0.0", vers: "v0.0.3"}, + {path: queryRepo, query: "v1.9.10-pre2+metadata", vers: "v1.9.10-pre2.0.20190513201126-42abcb6df8ee"}, + {path: queryRepo, query: "ed5ffdaa", vers: "v1.9.10-pre2.0.20191220134614-ed5ffdaa1f5e"}, + + // golang.org/issue/29262: The major version for a module without a suffix + // should be based on the most recent tag (v1 as appropriate, not v0 + // unconditionally). + {path: queryRepo, query: "42abcb6df8ee", vers: "v1.9.10-pre2.0.20190513201126-42abcb6df8ee"}, + + {path: queryRepo, query: "v1.9.10-pre2+wrongmetadata", err: `vcs-test.golang.org/git/querytest.git@v1.9.10-pre2+wrongmetadata: invalid version: unknown revision v1.9.10-pre2+wrongmetadata`}, + {path: queryRepo, query: "v1.9.10-pre2", err: `vcs-test.golang.org/git/querytest.git@v1.9.10-pre2: invalid version: unknown revision v1.9.10-pre2`}, + {path: queryRepo, query: "latest", vers: "v1.9.9"}, + {path: queryRepo, query: "latest", current: "v1.9.10-pre1", vers: "v1.9.9"}, + {path: queryRepo, query: "upgrade", vers: "v1.9.9"}, + {path: queryRepo, query: "upgrade", current: "v1.9.10-pre1", vers: "v1.9.10-pre1"}, + {path: queryRepo, query: "upgrade", current: "v1.9.10-pre2+metadata", vers: "v1.9.10-pre2.0.20190513201126-42abcb6df8ee"}, + {path: queryRepo, query: "upgrade", current: "v0.0.0-20190513201126-42abcb6df8ee", vers: "v0.0.0-20190513201126-42abcb6df8ee"}, + {path: queryRepo, query: "upgrade", allow: "NOMATCH", err: `no matching versions for query "upgrade"`}, + {path: queryRepo, query: "upgrade", current: "v1.9.9", allow: "NOMATCH", err: `vcs-test.golang.org/git/querytest.git@v1.9.9: disallowed module version`}, + {path: queryRepo, query: "upgrade", current: "v1.99.99", err: `vcs-test.golang.org/git/querytest.git@v1.99.99: invalid version: unknown revision v1.99.99`}, + {path: queryRepo, query: "patch", current: "", err: `can't query version "patch" of module vcs-test.golang.org/git/querytest.git: no existing version is required`}, + {path: queryRepo, query: "patch", current: "v0.1.0", vers: "v0.1.2"}, + {path: queryRepo, query: "patch", current: "v1.9.0", vers: "v1.9.9"}, + {path: queryRepo, query: "patch", current: "v1.9.10-pre1", vers: "v1.9.10-pre1"}, + {path: queryRepo, query: "patch", current: "v1.9.10-pre2+metadata", vers: "v1.9.10-pre2.0.20190513201126-42abcb6df8ee"}, + {path: queryRepo, query: "patch", current: "v1.99.99", err: `vcs-test.golang.org/git/querytest.git@v1.99.99: invalid version: unknown revision v1.99.99`}, + {path: queryRepo, query: ">v1.9.9", vers: "v1.9.10-pre1"}, + {path: queryRepo, query: ">v1.10.0", err: `no matching versions for query ">v1.10.0"`}, + {path: queryRepo, query: ">=v1.10.0", err: `no matching versions for query ">=v1.10.0"`}, + {path: queryRepo, query: "6cf84eb", vers: "v0.0.2-0.20180704023347-6cf84ebaea54"}, + + // golang.org/issue/27173: A pseudo-version may be based on the highest tag on + // any parent commit, or any existing semantically-lower tag: a given commit + // could have been a pre-release for a backport tag at any point. + {path: queryRepo, query: "3ef0cec634e0", vers: "v0.1.2-0.20180704023347-3ef0cec634e0"}, + {path: queryRepo, query: "v0.1.2-0.20180704023347-3ef0cec634e0", vers: "v0.1.2-0.20180704023347-3ef0cec634e0"}, + {path: queryRepo, query: "v0.1.1-0.20180704023347-3ef0cec634e0", vers: "v0.1.1-0.20180704023347-3ef0cec634e0"}, + {path: queryRepo, query: "v0.0.4-0.20180704023347-3ef0cec634e0", vers: "v0.0.4-0.20180704023347-3ef0cec634e0"}, + + // Invalid tags are tested in cmd/go/testdata/script/mod_pseudo_invalid.txt. + + {path: queryRepo, query: "start", vers: "v0.0.0-20180704023101-5e9e31667ddf"}, + {path: queryRepo, query: "5e9e31667ddf", vers: "v0.0.0-20180704023101-5e9e31667ddf"}, + {path: queryRepo, query: "v0.0.0-20180704023101-5e9e31667ddf", vers: "v0.0.0-20180704023101-5e9e31667ddf"}, + + {path: queryRepo, query: "7a1b6bf", vers: "v0.1.0"}, + + {path: queryRepoV2, query: "<v0.0.0", err: `no matching versions for query "<v0.0.0"`}, + {path: queryRepoV2, query: "<=v0.0.0", err: `no matching versions for query "<=v0.0.0"`}, + {path: queryRepoV2, query: ">v0.0.0", vers: "v2.0.0"}, + {path: queryRepoV2, query: ">=v0.0.0", vers: "v2.0.0"}, + + {path: queryRepoV2, query: "v2", vers: "v2.5.5"}, + {path: queryRepoV2, query: "v2.5", vers: "v2.5.5"}, + {path: queryRepoV2, query: "v2.6", err: `no matching versions for query "v2.6"`}, + {path: queryRepoV2, query: "v2.6.0-pre1", vers: "v2.6.0-pre1"}, + {path: queryRepoV2, query: "latest", vers: "v2.5.5"}, + + // Commit e0cf3de987e6 is actually v1.19.10-pre1, not anything resembling v3, + // and it has a go.mod file with a non-v3 module path. Attempting to query it + // as the v3 module should fail. + {path: queryRepoV3, query: "e0cf3de987e6", err: `vcs-test.golang.org/git/querytest.git/v3@v3.0.0-20180704024501-e0cf3de987e6: invalid version: go.mod has non-.../v3 module path "vcs-test.golang.org/git/querytest.git" (and .../v3/go.mod does not exist) at revision e0cf3de987e6`}, + + // The querytest repo does not have any commits tagged with major version 3, + // and the latest commit in the repo has a go.mod file specifying a non-v3 path. + // That should prevent us from resolving any version for the /v3 path. + {path: queryRepoV3, query: "latest", err: `no matching versions for query "latest"`}, + + {path: emptyRepoPath, query: "latest", vers: "v0.0.0-20180704023549-7bb914627242"}, + {path: emptyRepoPath, query: ">v0.0.0", err: `no matching versions for query ">v0.0.0"`}, + {path: emptyRepoPath, query: "<v10.0.0", err: `no matching versions for query "<v10.0.0"`}, +} + +func TestQuery(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + testenv.MustHaveExecPath(t, "git") + + ctx := context.Background() + + for _, tt := range queryTests { + allow := tt.allow + if allow == "" { + allow = "*" + } + allowed := func(ctx context.Context, m module.Version) error { + if ok, _ := path.Match(allow, m.Version); !ok { + return module.VersionError(m, ErrDisallowed) + } + return nil + } + tt := tt + t.Run(strings.ReplaceAll(tt.path, "/", "_")+"/"+tt.query+"/"+tt.current+"/"+allow, func(t *testing.T) { + t.Parallel() + + info, err := Query(ctx, tt.path, tt.query, tt.current, allowed) + if tt.err != "" { + if err == nil { + t.Errorf("Query(_, %q, %q, %q, %v) = %v, want error %q", tt.path, tt.query, tt.current, allow, info.Version, tt.err) + } else if err.Error() != tt.err { + t.Errorf("Query(_, %q, %q, %q, %v): %v\nwant error %q", tt.path, tt.query, tt.current, allow, err, tt.err) + } + return + } + if err != nil { + t.Fatalf("Query(_, %q, %q, %q, %v): %v\nwant %v", tt.path, tt.query, tt.current, allow, err, tt.vers) + } + if info.Version != tt.vers { + t.Errorf("Query(_, %q, %q, %q, %v) = %v, want %v", tt.path, tt.query, tt.current, allow, info.Version, tt.vers) + } + }) + } +} diff --git a/src/cmd/go/internal/modload/search.go b/src/cmd/go/internal/modload/search.go new file mode 100644 index 0000000..d392b5b --- /dev/null +++ b/src/cmd/go/internal/modload/search.go @@ -0,0 +1,305 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "runtime" + "sort" + "strings" + "sync" + + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/gover" + "cmd/go/internal/imports" + "cmd/go/internal/modindex" + "cmd/go/internal/par" + "cmd/go/internal/search" + "cmd/go/internal/str" + "cmd/go/internal/trace" + "cmd/internal/pkgpattern" + + "golang.org/x/mod/module" +) + +type stdFilter int8 + +const ( + omitStd = stdFilter(iota) + includeStd +) + +// matchPackages is like m.MatchPackages, but uses a local variable (rather than +// a global) for tags, can include or exclude packages in the standard library, +// and is restricted to the given list of modules. +func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) { + ctx, span := trace.StartSpan(ctx, "modload.matchPackages") + defer span.Done() + + m.Pkgs = []string{} + + isMatch := func(string) bool { return true } + treeCanMatch := func(string) bool { return true } + if !m.IsMeta() { + isMatch = pkgpattern.MatchPattern(m.Pattern()) + treeCanMatch = pkgpattern.TreeCanMatchPattern(m.Pattern()) + } + + var mu sync.Mutex + have := map[string]bool{ + "builtin": true, // ignore pseudo-package that exists only for documentation + } + addPkg := func(p string) { + mu.Lock() + m.Pkgs = append(m.Pkgs, p) + mu.Unlock() + } + if !cfg.BuildContext.CgoEnabled { + have["runtime/cgo"] = true // ignore during walk + } + + type pruning int8 + const ( + pruneVendor = pruning(1 << iota) + pruneGoMod + ) + + q := par.NewQueue(runtime.GOMAXPROCS(0)) + + walkPkgs := func(root, importPathRoot string, prune pruning) { + _, span := trace.StartSpan(ctx, "walkPkgs "+root) + defer span.Done() + + // If the root itself is a symlink to a directory, + // we want to follow it (see https://go.dev/issue/50807). + // Add a trailing separator to force that to happen. + root = str.WithFilePathSeparator(filepath.Clean(root)) + err := fsys.Walk(root, func(pkgDir string, fi fs.FileInfo, err error) error { + if err != nil { + m.AddError(err) + return nil + } + + want := true + elem := "" + + // Don't use GOROOT/src but do walk down into it. + if pkgDir == root { + if importPathRoot == "" { + return nil + } + } else { + // Avoid .foo, _foo, and testdata subdirectory trees. + _, elem = filepath.Split(pkgDir) + if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" { + want = false + } + } + + name := path.Join(importPathRoot, filepath.ToSlash(pkgDir[len(root):])) + if !treeCanMatch(name) { + want = false + } + + if !fi.IsDir() { + if fi.Mode()&fs.ModeSymlink != 0 && want && strings.Contains(m.Pattern(), "...") { + if target, err := fsys.Stat(pkgDir); err == nil && target.IsDir() { + fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", pkgDir) + } + } + return nil + } + + if !want { + return filepath.SkipDir + } + // Stop at module boundaries. + if (prune&pruneGoMod != 0) && pkgDir != root { + if fi, err := os.Stat(filepath.Join(pkgDir, "go.mod")); err == nil && !fi.IsDir() { + return filepath.SkipDir + } + } + + if !have[name] { + have[name] = true + if isMatch(name) { + q.Add(func() { + if _, _, err := scanDir(root, pkgDir, tags); err != imports.ErrNoGo { + addPkg(name) + } + }) + } + } + + if elem == "vendor" && (prune&pruneVendor != 0) { + return filepath.SkipDir + } + return nil + }) + if err != nil { + m.AddError(err) + } + } + + // Wait for all in-flight operations to complete before returning. + defer func() { + <-q.Idle() + sort.Strings(m.Pkgs) // sort everything we added for determinism + }() + + if filter == includeStd { + walkPkgs(cfg.GOROOTsrc, "", pruneGoMod) + if treeCanMatch("cmd") { + walkPkgs(filepath.Join(cfg.GOROOTsrc, "cmd"), "cmd", pruneGoMod) + } + } + + if cfg.BuildMod == "vendor" { + for _, mod := range MainModules.Versions() { + if modRoot := MainModules.ModRoot(mod); modRoot != "" { + walkPkgs(modRoot, MainModules.PathPrefix(mod), pruneGoMod|pruneVendor) + } + } + if HasModRoot() { + walkPkgs(VendorDir(), "", pruneVendor) + } + return + } + + for _, mod := range modules { + if gover.IsToolchain(mod.Path) || !treeCanMatch(mod.Path) { + continue + } + + var ( + root, modPrefix string + isLocal bool + ) + if MainModules.Contains(mod.Path) { + if MainModules.ModRoot(mod) == "" { + continue // If there is no main module, we can't search in it. + } + root = MainModules.ModRoot(mod) + modPrefix = MainModules.PathPrefix(mod) + isLocal = true + } else { + var err error + root, isLocal, err = fetch(ctx, mod) + if err != nil { + m.AddError(err) + continue + } + modPrefix = mod.Path + } + if mi, err := modindex.GetModule(root); err == nil { + walkFromIndex(mi, modPrefix, isMatch, treeCanMatch, tags, have, addPkg) + continue + } else if !errors.Is(err, modindex.ErrNotIndexed) { + m.AddError(err) + } + + prune := pruneVendor + if isLocal { + prune |= pruneGoMod + } + walkPkgs(root, modPrefix, prune) + } +} + +// walkFromIndex matches packages in a module using the module index. modroot +// is the module's root directory on disk, index is the modindex.Module for the +// module, and importPathRoot is the module's path prefix. +func walkFromIndex(index *modindex.Module, importPathRoot string, isMatch, treeCanMatch func(string) bool, tags, have map[string]bool, addPkg func(string)) { + index.Walk(func(reldir string) { + // Avoid .foo, _foo, and testdata subdirectory trees. + p := reldir + for { + elem, rest, found := strings.Cut(p, string(filepath.Separator)) + if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" { + return + } + if found && elem == "vendor" { + // Ignore this path if it contains the element "vendor" anywhere + // except for the last element (packages named vendor are allowed + // for historical reasons). Note that found is true when this + // isn't the last path element. + return + } + if !found { + // Didn't find the separator, so we're considering the last element. + break + } + p = rest + } + + // Don't use GOROOT/src. + if reldir == "" && importPathRoot == "" { + return + } + + name := path.Join(importPathRoot, filepath.ToSlash(reldir)) + if !treeCanMatch(name) { + return + } + + if !have[name] { + have[name] = true + if isMatch(name) { + if _, _, err := index.Package(reldir).ScanDir(tags); err != imports.ErrNoGo { + addPkg(name) + } + } + } + }) +} + +// MatchInModule identifies the packages matching the given pattern within the +// given module version, which does not need to be in the build list or module +// requirement graph. +// +// If m is the zero module.Version, MatchInModule matches the pattern +// against the standard library (std and cmd) in GOROOT/src. +func MatchInModule(ctx context.Context, pattern string, m module.Version, tags map[string]bool) *search.Match { + match := search.NewMatch(pattern) + if m == (module.Version{}) { + matchPackages(ctx, match, tags, includeStd, nil) + } + + LoadModFile(ctx) // Sets Target, needed by fetch and matchPackages. + + if !match.IsLiteral() { + matchPackages(ctx, match, tags, omitStd, []module.Version{m}) + return match + } + + root, isLocal, err := fetch(ctx, m) + if err != nil { + match.Errs = []error{err} + return match + } + + dir, haveGoFiles, err := dirInModule(pattern, m.Path, root, isLocal) + if err != nil { + match.Errs = []error{err} + return match + } + if haveGoFiles { + if _, _, err := scanDir(root, dir, tags); err != imports.ErrNoGo { + // ErrNoGo indicates that the directory is not actually a Go package, + // perhaps due to the tags in use. Any other non-nil error indicates a + // problem with one or more of the Go source files, but such an error does + // not stop the package from existing, so it has no impact on matching. + match.Pkgs = []string{pattern} + } + } + return match +} diff --git a/src/cmd/go/internal/modload/stat_openfile.go b/src/cmd/go/internal/modload/stat_openfile.go new file mode 100644 index 0000000..5773073 --- /dev/null +++ b/src/cmd/go/internal/modload/stat_openfile.go @@ -0,0 +1,28 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (js && wasm) || plan9 + +// On plan9, per http://9p.io/magic/man2html/2/access: “Since file permissions +// are checked by the server and group information is not known to the client, +// access must open the file to check permissions.” +// +// js,wasm is similar, in that it does not define syscall.Access. + +package modload + +import ( + "io/fs" + "os" +) + +// hasWritePerm reports whether the current user has permission to write to the +// file with the given info. +func hasWritePerm(path string, _ fs.FileInfo) bool { + if f, err := os.OpenFile(path, os.O_WRONLY, 0); err == nil { + f.Close() + return true + } + return false +} diff --git a/src/cmd/go/internal/modload/stat_unix.go b/src/cmd/go/internal/modload/stat_unix.go new file mode 100644 index 0000000..a0d5f4d --- /dev/null +++ b/src/cmd/go/internal/modload/stat_unix.go @@ -0,0 +1,32 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix + +package modload + +import ( + "io/fs" + "os" + "syscall" +) + +// hasWritePerm reports whether the current user has permission to write to the +// file with the given info. +// +// Although the root user on most Unix systems can write to files even without +// permission, hasWritePerm reports false if no appropriate permission bit is +// set even if the current user is root. +func hasWritePerm(path string, fi fs.FileInfo) bool { + if os.Getuid() == 0 { + // The root user can access any file, but we still want to default to + // read-only mode if the go.mod file is marked as globally non-writable. + // (If the user really intends not to be in readonly mode, they can + // pass -mod=mod explicitly.) + return fi.Mode()&0222 != 0 + } + + const W_OK = 0x2 + return syscall.Access(path, W_OK) == nil +} diff --git a/src/cmd/go/internal/modload/stat_windows.go b/src/cmd/go/internal/modload/stat_windows.go new file mode 100644 index 0000000..f29a991 --- /dev/null +++ b/src/cmd/go/internal/modload/stat_windows.go @@ -0,0 +1,21 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build windows + +package modload + +import "io/fs" + +// hasWritePerm reports whether the current user has permission to write to the +// file with the given info. +func hasWritePerm(_ string, fi fs.FileInfo) bool { + // Windows has a read-only attribute independent of ACLs, so use that to + // determine whether the file is intended to be overwritten. + // + // Per https://golang.org/pkg/os/#Chmod: + // “On Windows, only the 0200 bit (owner writable) of mode is used; it + // controls whether the file's read-only attribute is set or cleared.” + return fi.Mode()&0200 != 0 +} diff --git a/src/cmd/go/internal/modload/vendor.go b/src/cmd/go/internal/modload/vendor.go new file mode 100644 index 0000000..b2cb441 --- /dev/null +++ b/src/cmd/go/internal/modload/vendor.go @@ -0,0 +1,284 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "sync" + + "cmd/go/internal/base" + "cmd/go/internal/gover" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +var ( + vendorOnce sync.Once + vendorList []module.Version // modules that contribute packages to the build, in order of appearance + vendorReplaced []module.Version // all replaced modules; may or may not also contribute packages + vendorVersion map[string]string // module path → selected version (if known) + vendorPkgModule map[string]module.Version // package → containing module + vendorMeta map[module.Version]vendorMetadata +) + +type vendorMetadata struct { + Explicit bool + Replacement module.Version + GoVersion string +} + +// readVendorList reads the list of vendored modules from vendor/modules.txt. +func readVendorList(vendorDir string) { + vendorOnce.Do(func() { + vendorList = nil + vendorPkgModule = make(map[string]module.Version) + vendorVersion = make(map[string]string) + vendorMeta = make(map[module.Version]vendorMetadata) + vendorFile := filepath.Join(vendorDir, "modules.txt") + data, err := os.ReadFile(vendorFile) + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + base.Fatalf("go: %s", err) + } + return + } + + var mod module.Version + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, "# ") { + f := strings.Fields(line) + + if len(f) < 3 { + continue + } + if semver.IsValid(f[2]) { + // A module, but we don't yet know whether it is in the build list or + // only included to indicate a replacement. + mod = module.Version{Path: f[1], Version: f[2]} + f = f[3:] + } else if f[2] == "=>" { + // A wildcard replacement found in the main module's go.mod file. + mod = module.Version{Path: f[1]} + f = f[2:] + } else { + // Not a version or a wildcard replacement. + // We don't know how to interpret this module line, so ignore it. + mod = module.Version{} + continue + } + + if len(f) >= 2 && f[0] == "=>" { + meta := vendorMeta[mod] + if len(f) == 2 { + // File replacement. + meta.Replacement = module.Version{Path: f[1]} + vendorReplaced = append(vendorReplaced, mod) + } else if len(f) == 3 && semver.IsValid(f[2]) { + // Path and version replacement. + meta.Replacement = module.Version{Path: f[1], Version: f[2]} + vendorReplaced = append(vendorReplaced, mod) + } else { + // We don't understand this replacement. Ignore it. + } + vendorMeta[mod] = meta + } + continue + } + + // Not a module line. Must be a package within a module or a metadata + // directive, either of which requires a preceding module line. + if mod.Path == "" { + continue + } + + if annotations, ok := strings.CutPrefix(line, "## "); ok { + // Metadata. Take the union of annotations across multiple lines, if present. + meta := vendorMeta[mod] + for _, entry := range strings.Split(annotations, ";") { + entry = strings.TrimSpace(entry) + if entry == "explicit" { + meta.Explicit = true + } + if goVersion, ok := strings.CutPrefix(entry, "go "); ok { + meta.GoVersion = goVersion + rawGoVersion.Store(mod, meta.GoVersion) + if gover.Compare(goVersion, gover.Local()) > 0 { + base.Fatal(&gover.TooNewError{What: mod.Path + " in " + base.ShortPath(vendorFile), GoVersion: goVersion}) + } + } + // All other tokens are reserved for future use. + } + vendorMeta[mod] = meta + continue + } + + if f := strings.Fields(line); len(f) == 1 && module.CheckImportPath(f[0]) == nil { + // A package within the current module. + vendorPkgModule[f[0]] = mod + + // Since this module provides a package for the build, we know that it + // is in the build list and is the selected version of its path. + // If this information is new, record it. + if v, ok := vendorVersion[mod.Path]; !ok || gover.ModCompare(mod.Path, v, mod.Version) < 0 { + vendorList = append(vendorList, mod) + vendorVersion[mod.Path] = mod.Version + } + } + } + }) +} + +// checkVendorConsistency verifies that the vendor/modules.txt file matches (if +// go 1.14) or at least does not contradict (go 1.13 or earlier) the +// requirements and replacements listed in the main module's go.mod file. +func checkVendorConsistency(indexes []*modFileIndex, modFiles []*modfile.File, modRoots []string) { + // readVendorList only needs the main module to get the directory + // the vendor directory is in. + readVendorList(VendorDir()) + + if len(modFiles) < 1 { + // We should never get here if there are zero modfiles. Either + // we're in single module mode and there's a single module, or + // we're in workspace mode, and we fail earlier reporting that + // "no modules were found in the current workspace". + panic("checkVendorConsistency called with zero modfiles") + } + + pre114 := false + if !inWorkspaceMode() { // workspace mode was added after Go 1.14 + if len(indexes) != 1 { + panic(fmt.Errorf("not in workspace mode but number of indexes is %v, not 1", len(indexes))) + } + index := indexes[0] + if gover.Compare(index.goVersion, "1.14") < 0 { + // Go versions before 1.14 did not include enough information in + // vendor/modules.txt to check for consistency. + // If we know that we're on an earlier version, relax the consistency check. + pre114 = true + } + } + + vendErrors := new(strings.Builder) + vendErrorf := func(mod module.Version, format string, args ...any) { + detail := fmt.Sprintf(format, args...) + if mod.Version == "" { + fmt.Fprintf(vendErrors, "\n\t%s: %s", mod.Path, detail) + } else { + fmt.Fprintf(vendErrors, "\n\t%s@%s: %s", mod.Path, mod.Version, detail) + } + } + + // Iterate over the Require directives in their original (not indexed) order + // so that the errors match the original file. + for _, modFile := range modFiles { + for _, r := range modFile.Require { + if !vendorMeta[r.Mod].Explicit { + if pre114 { + // Before 1.14, modules.txt did not indicate whether modules were listed + // explicitly in the main module's go.mod file. + // However, we can at least detect a version mismatch if packages were + // vendored from a non-matching version. + if vv, ok := vendorVersion[r.Mod.Path]; ok && vv != r.Mod.Version { + vendErrorf(r.Mod, fmt.Sprintf("is explicitly required in go.mod, but vendor/modules.txt indicates %s@%s", r.Mod.Path, vv)) + } + } else { + vendErrorf(r.Mod, "is explicitly required in go.mod, but not marked as explicit in vendor/modules.txt") + } + } + } + } + + describe := func(m module.Version) string { + if m.Version == "" { + return m.Path + } + return m.Path + "@" + m.Version + } + + // We need to verify *all* replacements that occur in modfile: even if they + // don't directly apply to any module in the vendor list, the replacement + // go.mod file can affect the selected versions of other (transitive) + // dependencies + seenrep := make(map[module.Version]bool) + checkReplace := func(replaces []*modfile.Replace) { + for _, r := range replaces { + if seenrep[r.Old] { + continue // Don't print the same error more than once + } + seenrep[r.Old] = true + rNew, modRoot, replacementSource := replacementFrom(r.Old) + rNewCanonical := canonicalizeReplacePath(rNew, modRoot) + vr := vendorMeta[r.Old].Replacement + if vr == (module.Version{}) { + if rNewCanonical == (module.Version{}) { + // r.Old is not actually replaced. It might be a main module. + // Don't return an error. + } else if pre114 && (r.Old.Version == "" || vendorVersion[r.Old.Path] != r.Old.Version) { + // Before 1.14, modules.txt omitted wildcard replacements and + // replacements for modules that did not have any packages to vendor. + } else { + vendErrorf(r.Old, "is replaced in %s, but not marked as replaced in vendor/modules.txt", base.ShortPath(replacementSource)) + } + } else if vr != rNewCanonical { + vendErrorf(r.Old, "is replaced by %s in %s, but marked as replaced by %s in vendor/modules.txt", describe(rNew), base.ShortPath(replacementSource), describe(vr)) + } + } + } + for _, modFile := range modFiles { + checkReplace(modFile.Replace) + } + if MainModules.workFile != nil { + checkReplace(MainModules.workFile.Replace) + } + + for _, mod := range vendorList { + meta := vendorMeta[mod] + if meta.Explicit { + // in workspace mode, check that it's required by at least one of the main modules + var foundRequire bool + for _, index := range indexes { + if _, inGoMod := index.require[mod]; inGoMod { + foundRequire = true + } + } + if !foundRequire { + article := "" + if inWorkspaceMode() { + article = "a " + } + vendErrorf(mod, "is marked as explicit in vendor/modules.txt, but not explicitly required in %vgo.mod", article) + } + + } + } + + for _, mod := range vendorReplaced { + r := Replacement(mod) + replacementSource := "go.mod" + if inWorkspaceMode() { + replacementSource = "the workspace" + } + if r == (module.Version{}) { + vendErrorf(mod, "is marked as replaced in vendor/modules.txt, but not replaced in %s", replacementSource) + continue + } + // If both replacements exist, we've already reported that they're different above. + } + + if vendErrors.Len() > 0 { + subcmd := "mod" + if inWorkspaceMode() { + subcmd = "work" + } + base.Fatalf("go: inconsistent vendoring in %s:%s\n\n\tTo ignore the vendor directory, use -mod=readonly or -mod=mod.\n\tTo sync the vendor directory, run:\n\t\tgo %s vendor", filepath.Dir(VendorDir()), vendErrors, subcmd) + } +} |