diff options
Diffstat (limited to 'src/cmd/go/internal/modcmd')
| -rw-r--r-- | src/cmd/go/internal/modcmd/download.go | 389 | ||||
| -rw-r--r-- | src/cmd/go/internal/modcmd/edit.go | 546 | ||||
| -rw-r--r-- | src/cmd/go/internal/modcmd/graph.go | 96 | ||||
| -rw-r--r-- | src/cmd/go/internal/modcmd/init.go | 49 | ||||
| -rw-r--r-- | src/cmd/go/internal/modcmd/mod.go | 33 | ||||
| -rw-r--r-- | src/cmd/go/internal/modcmd/tidy.go | 139 | ||||
| -rw-r--r-- | src/cmd/go/internal/modcmd/vendor.go | 481 | ||||
| -rw-r--r-- | src/cmd/go/internal/modcmd/verify.go | 143 | ||||
| -rw-r--r-- | src/cmd/go/internal/modcmd/why.go | 143 |
9 files changed, 2019 insertions, 0 deletions
diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go new file mode 100644 index 0000000..373acce --- /dev/null +++ b/src/cmd/go/internal/modcmd/download.go @@ -0,0 +1,389 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modcmd + +import ( + "context" + "encoding/json" + "errors" + "os" + "runtime" + "sync" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/gover" + "cmd/go/internal/modfetch" + "cmd/go/internal/modfetch/codehost" + "cmd/go/internal/modload" + "cmd/go/internal/toolchain" + + "golang.org/x/mod/module" +) + +var cmdDownload = &base.Command{ + UsageLine: "go mod download [-x] [-json] [-reuse=old.json] [modules]", + Short: "download modules to local cache", + Long: ` +Download downloads the named modules, which can be module patterns selecting +dependencies of the main module or module queries of the form path@version. + +With no arguments, download applies to the modules needed to build and test +the packages in the main module: the modules explicitly required by the main +module if it is at 'go 1.17' or higher, or all transitively-required modules +if at 'go 1.16' or lower. + +The go command will automatically download modules as needed during ordinary +execution. The "go mod download" command is useful mainly for pre-filling +the local cache or to compute the answers for a Go module proxy. + +By default, download writes nothing to standard output. It may print progress +messages and errors to standard error. + +The -json flag causes download to print a sequence of JSON objects +to standard output, describing each downloaded module (or failure), +corresponding to this Go struct: + + type Module struct { + Path string // module path + Query string // version query corresponding to this version + Version string // module version + Error string // error loading module + Info string // absolute path to cached .info file + GoMod string // absolute path to cached .mod file + Zip string // absolute path to cached .zip file + Dir string // absolute path to cached source root directory + Sum string // checksum for path, version (as in go.sum) + GoModSum string // checksum for go.mod (as in go.sum) + Origin any // provenance of module + Reuse bool // reuse of old module info is safe + } + +The -reuse flag accepts the name of file containing the JSON output of a +previous 'go mod download -json' invocation. The go command may use this +file to determine that a module is unchanged since the previous invocation +and avoid redownloading it. Modules that are not redownloaded will be marked +in the new output by setting the Reuse field to true. Normally the module +cache provides this kind of reuse automatically; the -reuse flag can be +useful on systems that do not preserve the module cache. + +The -x flag causes download to print the commands download executes. + +See https://golang.org/ref/mod#go-mod-download for more about 'go mod download'. + +See https://golang.org/ref/mod#version-queries for more about version queries. + `, +} + +var ( + downloadJSON = cmdDownload.Flag.Bool("json", false, "") + downloadReuse = cmdDownload.Flag.String("reuse", "", "") +) + +func init() { + cmdDownload.Run = runDownload // break init cycle + + // TODO(jayconrod): https://golang.org/issue/35849 Apply -x to other 'go mod' commands. + cmdDownload.Flag.BoolVar(&cfg.BuildX, "x", false, "") + base.AddChdirFlag(&cmdDownload.Flag) + base.AddModCommonFlags(&cmdDownload.Flag) +} + +// A ModuleJSON describes the result of go mod download. +type ModuleJSON struct { + Path string `json:",omitempty"` + Version string `json:",omitempty"` + Query string `json:",omitempty"` + Error string `json:",omitempty"` + Info string `json:",omitempty"` + GoMod string `json:",omitempty"` + Zip string `json:",omitempty"` + Dir string `json:",omitempty"` + Sum string `json:",omitempty"` + GoModSum string `json:",omitempty"` + + Origin *codehost.Origin `json:",omitempty"` + Reuse bool `json:",omitempty"` +} + +func runDownload(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + + // Check whether modules are enabled and whether we're in a module. + modload.ForceUseModules = true + modload.ExplicitWriteGoMod = true + haveExplicitArgs := len(args) > 0 + + if modload.HasModRoot() || modload.WorkFilePath() != "" { + modload.LoadModFile(ctx) // to fill MainModules + + if haveExplicitArgs { + for _, mainModule := range modload.MainModules.Versions() { + targetAtUpgrade := mainModule.Path + "@upgrade" + targetAtPatch := mainModule.Path + "@patch" + for _, arg := range args { + switch arg { + case mainModule.Path, targetAtUpgrade, targetAtPatch: + os.Stderr.WriteString("go: skipping download of " + arg + " that resolves to the main module\n") + } + } + } + } else if modload.WorkFilePath() != "" { + // TODO(#44435): Think about what the correct query is to download the + // right set of modules. Also see code review comment at + // https://go-review.googlesource.com/c/go/+/359794/comments/ce946a80_6cf53992. + args = []string{"all"} + } else { + mainModule := modload.MainModules.Versions()[0] + modFile := modload.MainModules.ModFile(mainModule) + if modFile.Go == nil || gover.Compare(modFile.Go.Version, gover.ExplicitIndirectVersion) < 0 { + if len(modFile.Require) > 0 { + args = []string{"all"} + } + } else { + // As of Go 1.17, the go.mod file explicitly requires every module + // that provides any package imported by the main module. + // 'go mod download' is typically run before testing packages in the + // main module, so by default we shouldn't download the others + // (which are presumed irrelevant to the packages in the main module). + // See https://golang.org/issue/44435. + // + // However, we also need to load the full module graph, to ensure that + // we have downloaded enough of the module graph to run 'go list all', + // 'go mod graph', and similar commands. + _, err := modload.LoadModGraph(ctx, "") + if err != nil { + // TODO(#64008): call base.Fatalf instead of toolchain.SwitchOrFatal + // here, since we can only reach this point with an outdated toolchain + // if the go.mod file is inconsistent. + toolchain.SwitchOrFatal(ctx, err) + } + + for _, m := range modFile.Require { + args = append(args, m.Mod.Path) + } + } + } + } + + if len(args) == 0 { + if modload.HasModRoot() { + os.Stderr.WriteString("go: no module dependencies to download\n") + } else { + base.Errorf("go: no modules specified (see 'go help mod download')") + } + base.Exit() + } + + if *downloadReuse != "" && modload.HasModRoot() { + base.Fatalf("go mod download -reuse cannot be used inside a module") + } + + var mods []*ModuleJSON + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + infos, infosErr := modload.ListModules(ctx, args, 0, *downloadReuse) + + // There is a bit of a chicken-and-egg problem here: ideally we need to know + // which Go version to switch to to download the requested modules, but if we + // haven't downloaded the module's go.mod file yet the GoVersion field of its + // info struct is not yet populated. + // + // We also need to be careful to only print the info for each module once + // if the -json flag is set. + // + // In theory we could go through each module in the list, attempt to download + // its go.mod file, and record the maximum version (either from the file or + // from the resulting TooNewError), all before we try the actual full download + // of each module. + // + // For now, we go ahead and try all the downloads and collect the errors, and + // if any download failed due to a TooNewError, we switch toolchains and try + // again. Any downloads that already succeeded will still be in cache. + // That won't give optimal concurrency (we'll do two batches of concurrent + // downloads instead of all in one batch), and it might add a little overhead + // to look up the downloads from the first batch in the module cache when + // we see them again in the second batch. On the other hand, it's way simpler + // to implement, and not really any more expensive if the user is requesting + // no explicit arguments (their go.mod file should already list an appropriate + // toolchain version) or only one module (as is used by the Go Module Proxy). + + if infosErr != nil { + var sw toolchain.Switcher + sw.Error(infosErr) + if sw.NeedSwitch() { + sw.Switch(ctx) + } + // Otherwise, wait to report infosErr after we have downloaded + // when we can. + } + + if !haveExplicitArgs && modload.WorkFilePath() == "" { + // 'go mod download' is sometimes run without arguments to pre-populate the + // module cache. In modules that aren't at go 1.17 or higher, it may fetch + // modules that aren't needed to build packages in the main module. This is + // usually not intended, so don't save sums for downloaded modules + // (golang.org/issue/45332). We do still fix inconsistencies in go.mod + // though. + // + // TODO(#64008): In the future, report an error if go.mod or go.sum need to + // be updated after loading the build list. This may require setting + // the mode to "mod" or "readonly" depending on haveExplicitArgs. + if err := modload.WriteGoMod(ctx, modload.WriteOpts{}); err != nil { + base.Fatal(err) + } + } + + var downloadErrs sync.Map + for _, info := range infos { + if info.Replace != nil { + info = info.Replace + } + if info.Version == "" && info.Error == nil { + // main module or module replaced with file path. + // Nothing to download. + continue + } + m := &ModuleJSON{ + Path: info.Path, + Version: info.Version, + Query: info.Query, + Reuse: info.Reuse, + Origin: info.Origin, + } + mods = append(mods, m) + if info.Error != nil { + m.Error = info.Error.Err + continue + } + if m.Reuse { + continue + } + sem <- token{} + go func() { + err := DownloadModule(ctx, m) + if err != nil { + downloadErrs.Store(m, err) + m.Error = err.Error() + } + <-sem + }() + } + + // Fill semaphore channel to wait for goroutines to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} + } + + // If there were explicit arguments + // (like 'go mod download golang.org/x/tools@latest'), + // check whether we need to upgrade the toolchain in order to download them. + // + // (If invoked without arguments, we expect the module graph to already + // be tidy and the go.mod file to declare a 'go' version that satisfies + // transitive requirements. If that invariant holds, then we should have + // already upgraded when we loaded the module graph, and should not need + // an additional check here. See https://go.dev/issue/45551.) + // + // We also allow upgrades if in a workspace because in workspace mode + // with no arguments we download the module pattern "all", + // which may include dependencies that are normally pruned out + // of the individual modules in the workspace. + if haveExplicitArgs || modload.WorkFilePath() != "" { + var sw toolchain.Switcher + // Add errors to the Switcher in deterministic order so that they will be + // logged deterministically. + for _, m := range mods { + if erri, ok := downloadErrs.Load(m); ok { + sw.Error(erri.(error)) + } + } + // Only call sw.Switch if it will actually switch. + // Otherwise, we may want to write the errors as JSON + // (instead of using base.Error as sw.Switch would), + // and we may also have other errors to report from the + // initial infos returned by ListModules. + if sw.NeedSwitch() { + sw.Switch(ctx) + } + } + + if *downloadJSON { + for _, m := range mods { + b, err := json.MarshalIndent(m, "", "\t") + if err != nil { + base.Fatal(err) + } + os.Stdout.Write(append(b, '\n')) + if m.Error != "" { + base.SetExitStatus(1) + } + } + } else { + for _, m := range mods { + if m.Error != "" { + base.Error(errors.New(m.Error)) + } + } + base.ExitIfErrors() + } + + // If there were explicit arguments, update go.mod and especially go.sum. + // 'go mod download mod@version' is a useful way to add a sum without using + // 'go get mod@version', which may have other side effects. We print this in + // some error message hints. + // + // If we're in workspace mode, update go.work.sum with checksums for all of + // the modules we downloaded that aren't already recorded. Since a requirement + // in one module may upgrade a dependency of another, we can't be sure that + // the import graph matches the import graph of any given module in isolation, + // so we may end up needing to load packages from modules that wouldn't + // otherwise be relevant. + // + // TODO(#44435): If we adjust the set of modules downloaded in workspace mode, + // we may also need to adjust the logic for saving checksums here. + // + // Don't save sums for 'go mod download' without arguments unless we're in + // workspace mode; see comment above. + if haveExplicitArgs || modload.WorkFilePath() != "" { + if err := modload.WriteGoMod(ctx, modload.WriteOpts{}); err != nil { + base.Error(err) + } + } + + // If there was an error matching some of the requested packages, emit it now + // (after we've written the checksums for the modules that were downloaded + // successfully). + if infosErr != nil { + base.Error(infosErr) + } +} + +// DownloadModule runs 'go mod download' for m.Path@m.Version, +// leaving the results (including any error) in m itself. +func DownloadModule(ctx context.Context, m *ModuleJSON) error { + var err error + _, file, err := modfetch.InfoFile(ctx, m.Path, m.Version) + if err != nil { + return err + } + m.Info = file + m.GoMod, err = modfetch.GoModFile(ctx, m.Path, m.Version) + if err != nil { + return err + } + m.GoModSum, err = modfetch.GoModSum(ctx, m.Path, m.Version) + if err != nil { + return err + } + mod := module.Version{Path: m.Path, Version: m.Version} + m.Zip, err = modfetch.DownloadZip(ctx, mod) + if err != nil { + return err + } + m.Sum = modfetch.Sum(ctx, mod) + m.Dir, err = modfetch.Download(ctx, mod) + return err +} diff --git a/src/cmd/go/internal/modcmd/edit.go b/src/cmd/go/internal/modcmd/edit.go new file mode 100644 index 0000000..db131b0 --- /dev/null +++ b/src/cmd/go/internal/modcmd/edit.go @@ -0,0 +1,546 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// go mod edit + +package modcmd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/gover" + "cmd/go/internal/lockedfile" + "cmd/go/internal/modfetch" + "cmd/go/internal/modload" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" +) + +var cmdEdit = &base.Command{ + UsageLine: "go mod edit [editing flags] [-fmt|-print|-json] [go.mod]", + Short: "edit go.mod from tools or scripts", + Long: ` +Edit provides a command-line interface for editing go.mod, +for use primarily by tools or scripts. It reads only go.mod; +it does not look up information about the modules involved. +By default, edit reads and writes the go.mod file of the main module, +but a different target file can be specified after the editing flags. + +The editing flags specify a sequence of editing operations. + +The -fmt flag reformats the go.mod file without making other changes. +This reformatting is also implied by any other modifications that use or +rewrite the go.mod file. The only time this flag is needed is if no other +flags are specified, as in 'go mod edit -fmt'. + +The -module flag changes the module's path (the go.mod file's module line). + +The -require=path@version and -droprequire=path flags +add and drop a requirement on the given module path and version. +Note that -require overrides any existing requirements on path. +These flags are mainly for tools that understand the module graph. +Users should prefer 'go get path@version' or 'go get path@none', +which make other go.mod adjustments as needed to satisfy +constraints imposed by other modules. + +The -exclude=path@version and -dropexclude=path@version flags +add and drop an exclusion for the given module path and version. +Note that -exclude=path@version is a no-op if that exclusion already exists. + +The -replace=old[@v]=new[@v] flag adds a replacement of the given +module path and version pair. If the @v in old@v is omitted, a +replacement without a version on the left side is added, which applies +to all versions of the old module path. If the @v in new@v is omitted, +the new path should be a local module root directory, not a module +path. Note that -replace overrides any redundant replacements for old[@v], +so omitting @v will drop existing replacements for specific versions. + +The -dropreplace=old[@v] flag drops a replacement of the given +module path and version pair. If the @v is omitted, a replacement without +a version on the left side is dropped. + +The -retract=version and -dropretract=version flags add and drop a +retraction on the given version. The version may be a single version +like "v1.2.3" or a closed interval like "[v1.1.0,v1.1.9]". Note that +-retract=version is a no-op if that retraction already exists. + +The -require, -droprequire, -exclude, -dropexclude, -replace, +-dropreplace, -retract, and -dropretract editing flags may be repeated, +and the changes are applied in the order given. + +The -go=version flag sets the expected Go language version. + +The -toolchain=name flag sets the Go toolchain to use. + +The -print flag prints the final go.mod in its text format instead of +writing it back to go.mod. + +The -json flag prints the final go.mod file in JSON format instead of +writing it back to go.mod. The JSON output corresponds to these Go types: + + type Module struct { + Path string + Version string + } + + type GoMod struct { + Module ModPath + Go string + Toolchain string + Require []Require + Exclude []Module + Replace []Replace + Retract []Retract + } + + type ModPath struct { + Path string + Deprecated string + } + + type Require struct { + Path string + Version string + Indirect bool + } + + type Replace struct { + Old Module + New Module + } + + type Retract struct { + Low string + High string + Rationale string + } + +Retract entries representing a single version (not an interval) will have +the "Low" and "High" fields set to the same value. + +Note that this only describes the go.mod file itself, not other modules +referred to indirectly. For the full set of modules available to a build, +use 'go list -m -json all'. + +Edit also provides the -C, -n, and -x build flags. + +See https://golang.org/ref/mod#go-mod-edit for more about 'go mod edit'. + `, +} + +var ( + editFmt = cmdEdit.Flag.Bool("fmt", false, "") + editGo = cmdEdit.Flag.String("go", "", "") + editToolchain = cmdEdit.Flag.String("toolchain", "", "") + editJSON = cmdEdit.Flag.Bool("json", false, "") + editPrint = cmdEdit.Flag.Bool("print", false, "") + editModule = cmdEdit.Flag.String("module", "", "") + edits []func(*modfile.File) // edits specified in flags +) + +type flagFunc func(string) + +func (f flagFunc) String() string { return "" } +func (f flagFunc) Set(s string) error { f(s); return nil } + +func init() { + cmdEdit.Run = runEdit // break init cycle + + cmdEdit.Flag.Var(flagFunc(flagRequire), "require", "") + cmdEdit.Flag.Var(flagFunc(flagDropRequire), "droprequire", "") + cmdEdit.Flag.Var(flagFunc(flagExclude), "exclude", "") + cmdEdit.Flag.Var(flagFunc(flagDropReplace), "dropreplace", "") + cmdEdit.Flag.Var(flagFunc(flagReplace), "replace", "") + cmdEdit.Flag.Var(flagFunc(flagDropExclude), "dropexclude", "") + cmdEdit.Flag.Var(flagFunc(flagRetract), "retract", "") + cmdEdit.Flag.Var(flagFunc(flagDropRetract), "dropretract", "") + + base.AddBuildFlagsNX(&cmdEdit.Flag) + base.AddChdirFlag(&cmdEdit.Flag) + base.AddModCommonFlags(&cmdEdit.Flag) +} + +func runEdit(ctx context.Context, cmd *base.Command, args []string) { + anyFlags := *editModule != "" || + *editGo != "" || + *editToolchain != "" || + *editJSON || + *editPrint || + *editFmt || + len(edits) > 0 + + if !anyFlags { + base.Fatalf("go: no flags specified (see 'go help mod edit').") + } + + if *editJSON && *editPrint { + base.Fatalf("go: cannot use both -json and -print") + } + + if len(args) > 1 { + base.Fatalf("go: too many arguments") + } + var gomod string + if len(args) == 1 { + gomod = args[0] + } else { + gomod = modload.ModFilePath() + } + + if *editModule != "" { + if err := module.CheckImportPath(*editModule); err != nil { + base.Fatalf("go: invalid -module: %v", err) + } + } + + if *editGo != "" && *editGo != "none" { + if !modfile.GoVersionRE.MatchString(*editGo) { + base.Fatalf(`go mod: invalid -go option; expecting something like "-go %s"`, gover.Local()) + } + } + if *editToolchain != "" && *editToolchain != "none" { + if !modfile.ToolchainRE.MatchString(*editToolchain) { + base.Fatalf(`go mod: invalid -toolchain option; expecting something like "-toolchain go%s"`, gover.Local()) + } + } + + data, err := lockedfile.Read(gomod) + if err != nil { + base.Fatal(err) + } + + modFile, err := modfile.Parse(gomod, data, nil) + if err != nil { + base.Fatalf("go: errors parsing %s:\n%s", base.ShortPath(gomod), err) + } + + if *editModule != "" { + modFile.AddModuleStmt(*editModule) + } + + if *editGo == "none" { + modFile.DropGoStmt() + } else if *editGo != "" { + if err := modFile.AddGoStmt(*editGo); err != nil { + base.Fatalf("go: internal error: %v", err) + } + } + if *editToolchain == "none" { + modFile.DropToolchainStmt() + } else if *editToolchain != "" { + if err := modFile.AddToolchainStmt(*editToolchain); err != nil { + base.Fatalf("go: internal error: %v", err) + } + } + + if len(edits) > 0 { + for _, edit := range edits { + edit(modFile) + } + } + modFile.SortBlocks() + modFile.Cleanup() // clean file after edits + + if *editJSON { + editPrintJSON(modFile) + return + } + + out, err := modFile.Format() + if err != nil { + base.Fatal(err) + } + + if *editPrint { + os.Stdout.Write(out) + return + } + + // Make a best-effort attempt to acquire the side lock, only to exclude + // previous versions of the 'go' command from making simultaneous edits. + if unlock, err := modfetch.SideLock(ctx); err == nil { + defer unlock() + } + + err = lockedfile.Transform(gomod, func(lockedData []byte) ([]byte, error) { + if !bytes.Equal(lockedData, data) { + return nil, errors.New("go.mod changed during editing; not overwriting") + } + return out, nil + }) + if err != nil { + base.Fatal(err) + } +} + +// parsePathVersion parses -flag=arg expecting arg to be path@version. +func parsePathVersion(flag, arg string) (path, version string) { + before, after, found := strings.Cut(arg, "@") + if !found { + base.Fatalf("go: -%s=%s: need path@version", flag, arg) + } + path, version = strings.TrimSpace(before), strings.TrimSpace(after) + if err := module.CheckImportPath(path); err != nil { + base.Fatalf("go: -%s=%s: invalid path: %v", flag, arg, err) + } + + if !allowedVersionArg(version) { + base.Fatalf("go: -%s=%s: invalid version %q", flag, arg, version) + } + + return path, version +} + +// parsePath parses -flag=arg expecting arg to be path (not path@version). +func parsePath(flag, arg string) (path string) { + if strings.Contains(arg, "@") { + base.Fatalf("go: -%s=%s: need just path, not path@version", flag, arg) + } + path = arg + if err := module.CheckImportPath(path); err != nil { + base.Fatalf("go: -%s=%s: invalid path: %v", flag, arg, err) + } + return path +} + +// parsePathVersionOptional parses path[@version], using adj to +// describe any errors. +func parsePathVersionOptional(adj, arg string, allowDirPath bool) (path, version string, err error) { + if allowDirPath && modfile.IsDirectoryPath(arg) { + return arg, "", nil + } + before, after, found := strings.Cut(arg, "@") + if !found { + path = arg + } else { + path, version = strings.TrimSpace(before), strings.TrimSpace(after) + } + if err := module.CheckImportPath(path); err != nil { + return path, version, fmt.Errorf("invalid %s path: %v", adj, err) + } + if path != arg && !allowedVersionArg(version) { + return path, version, fmt.Errorf("invalid %s version: %q", adj, version) + } + return path, version, nil +} + +// parseVersionInterval parses a single version like "v1.2.3" or a closed +// interval like "[v1.2.3,v1.4.5]". Note that a single version has the same +// representation as an interval with equal upper and lower bounds: both +// Low and High are set. +func parseVersionInterval(arg string) (modfile.VersionInterval, error) { + if !strings.HasPrefix(arg, "[") { + if !allowedVersionArg(arg) { + return modfile.VersionInterval{}, fmt.Errorf("invalid version: %q", arg) + } + return modfile.VersionInterval{Low: arg, High: arg}, nil + } + if !strings.HasSuffix(arg, "]") { + return modfile.VersionInterval{}, fmt.Errorf("invalid version interval: %q", arg) + } + s := arg[1 : len(arg)-1] + before, after, found := strings.Cut(s, ",") + if !found { + return modfile.VersionInterval{}, fmt.Errorf("invalid version interval: %q", arg) + } + low := strings.TrimSpace(before) + high := strings.TrimSpace(after) + if !allowedVersionArg(low) || !allowedVersionArg(high) { + return modfile.VersionInterval{}, fmt.Errorf("invalid version interval: %q", arg) + } + return modfile.VersionInterval{Low: low, High: high}, nil +} + +// allowedVersionArg returns whether a token may be used as a version in go.mod. +// We don't call modfile.CheckPathVersion, because that insists on versions +// being in semver form, but here we want to allow versions like "master" or +// "1234abcdef", which the go command will resolve the next time it runs (or +// during -fix). Even so, we need to make sure the version is a valid token. +func allowedVersionArg(arg string) bool { + return !modfile.MustQuote(arg) +} + +// flagRequire implements the -require flag. +func flagRequire(arg string) { + path, version := parsePathVersion("require", arg) + edits = append(edits, func(f *modfile.File) { + if err := f.AddRequire(path, version); err != nil { + base.Fatalf("go: -require=%s: %v", arg, err) + } + }) +} + +// flagDropRequire implements the -droprequire flag. +func flagDropRequire(arg string) { + path := parsePath("droprequire", arg) + edits = append(edits, func(f *modfile.File) { + if err := f.DropRequire(path); err != nil { + base.Fatalf("go: -droprequire=%s: %v", arg, err) + } + }) +} + +// flagExclude implements the -exclude flag. +func flagExclude(arg string) { + path, version := parsePathVersion("exclude", arg) + edits = append(edits, func(f *modfile.File) { + if err := f.AddExclude(path, version); err != nil { + base.Fatalf("go: -exclude=%s: %v", arg, err) + } + }) +} + +// flagDropExclude implements the -dropexclude flag. +func flagDropExclude(arg string) { + path, version := parsePathVersion("dropexclude", arg) + edits = append(edits, func(f *modfile.File) { + if err := f.DropExclude(path, version); err != nil { + base.Fatalf("go: -dropexclude=%s: %v", arg, err) + } + }) +} + +// flagReplace implements the -replace flag. +func flagReplace(arg string) { + before, after, found := strings.Cut(arg, "=") + if !found { + base.Fatalf("go: -replace=%s: need old[@v]=new[@w] (missing =)", arg) + } + old, new := strings.TrimSpace(before), strings.TrimSpace(after) + if strings.HasPrefix(new, ">") { + base.Fatalf("go: -replace=%s: separator between old and new is =, not =>", arg) + } + oldPath, oldVersion, err := parsePathVersionOptional("old", old, false) + if err != nil { + base.Fatalf("go: -replace=%s: %v", arg, err) + } + newPath, newVersion, err := parsePathVersionOptional("new", new, true) + if err != nil { + base.Fatalf("go: -replace=%s: %v", arg, err) + } + if newPath == new && !modfile.IsDirectoryPath(new) { + base.Fatalf("go: -replace=%s: unversioned new path must be local directory", arg) + } + + edits = append(edits, func(f *modfile.File) { + if err := f.AddReplace(oldPath, oldVersion, newPath, newVersion); err != nil { + base.Fatalf("go: -replace=%s: %v", arg, err) + } + }) +} + +// flagDropReplace implements the -dropreplace flag. +func flagDropReplace(arg string) { + path, version, err := parsePathVersionOptional("old", arg, true) + if err != nil { + base.Fatalf("go: -dropreplace=%s: %v", arg, err) + } + edits = append(edits, func(f *modfile.File) { + if err := f.DropReplace(path, version); err != nil { + base.Fatalf("go: -dropreplace=%s: %v", arg, err) + } + }) +} + +// flagRetract implements the -retract flag. +func flagRetract(arg string) { + vi, err := parseVersionInterval(arg) + if err != nil { + base.Fatalf("go: -retract=%s: %v", arg, err) + } + edits = append(edits, func(f *modfile.File) { + if err := f.AddRetract(vi, ""); err != nil { + base.Fatalf("go: -retract=%s: %v", arg, err) + } + }) +} + +// flagDropRetract implements the -dropretract flag. +func flagDropRetract(arg string) { + vi, err := parseVersionInterval(arg) + if err != nil { + base.Fatalf("go: -dropretract=%s: %v", arg, err) + } + edits = append(edits, func(f *modfile.File) { + if err := f.DropRetract(vi); err != nil { + base.Fatalf("go: -dropretract=%s: %v", arg, err) + } + }) +} + +// fileJSON is the -json output data structure. +type fileJSON struct { + Module editModuleJSON + Go string `json:",omitempty"` + Toolchain string `json:",omitempty"` + Require []requireJSON + Exclude []module.Version + Replace []replaceJSON + Retract []retractJSON +} + +type editModuleJSON struct { + Path string + Deprecated string `json:",omitempty"` +} + +type requireJSON struct { + Path string + Version string `json:",omitempty"` + Indirect bool `json:",omitempty"` +} + +type replaceJSON struct { + Old module.Version + New module.Version +} + +type retractJSON struct { + Low string `json:",omitempty"` + High string `json:",omitempty"` + Rationale string `json:",omitempty"` +} + +// editPrintJSON prints the -json output. +func editPrintJSON(modFile *modfile.File) { + var f fileJSON + if modFile.Module != nil { + f.Module = editModuleJSON{ + Path: modFile.Module.Mod.Path, + Deprecated: modFile.Module.Deprecated, + } + } + if modFile.Go != nil { + f.Go = modFile.Go.Version + } + if modFile.Toolchain != nil { + f.Toolchain = modFile.Toolchain.Name + } + for _, r := range modFile.Require { + f.Require = append(f.Require, requireJSON{Path: r.Mod.Path, Version: r.Mod.Version, Indirect: r.Indirect}) + } + for _, x := range modFile.Exclude { + f.Exclude = append(f.Exclude, x.Mod) + } + for _, r := range modFile.Replace { + f.Replace = append(f.Replace, replaceJSON{r.Old, r.New}) + } + for _, r := range modFile.Retract { + f.Retract = append(f.Retract, retractJSON{r.Low, r.High, r.Rationale}) + } + data, err := json.MarshalIndent(&f, "", "\t") + if err != nil { + base.Fatalf("go: internal error: %v", err) + } + data = append(data, '\n') + os.Stdout.Write(data) +} diff --git a/src/cmd/go/internal/modcmd/graph.go b/src/cmd/go/internal/modcmd/graph.go new file mode 100644 index 0000000..172c1dd --- /dev/null +++ b/src/cmd/go/internal/modcmd/graph.go @@ -0,0 +1,96 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// go mod graph + +package modcmd + +import ( + "bufio" + "context" + "os" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/gover" + "cmd/go/internal/modload" + "cmd/go/internal/toolchain" + + "golang.org/x/mod/module" +) + +var cmdGraph = &base.Command{ + UsageLine: "go mod graph [-go=version] [-x]", + Short: "print module requirement graph", + Long: ` +Graph prints the module requirement graph (with replacements applied) +in text form. Each line in the output has two space-separated fields: a module +and one of its requirements. Each module is identified as a string of the form +path@version, except for the main module, which has no @version suffix. + +The -go flag causes graph to report the module graph as loaded by the +given Go version, instead of the version indicated by the 'go' directive +in the go.mod file. + +The -x flag causes graph to print the commands graph executes. + +See https://golang.org/ref/mod#go-mod-graph for more about 'go mod graph'. + `, + Run: runGraph, +} + +var ( + graphGo goVersionFlag +) + +func init() { + cmdGraph.Flag.Var(&graphGo, "go", "") + cmdGraph.Flag.BoolVar(&cfg.BuildX, "x", false, "") + base.AddChdirFlag(&cmdGraph.Flag) + base.AddModCommonFlags(&cmdGraph.Flag) +} + +func runGraph(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + + if len(args) > 0 { + base.Fatalf("go: 'go mod graph' accepts no arguments") + } + modload.ForceUseModules = true + modload.RootMode = modload.NeedRoot + + goVersion := graphGo.String() + if goVersion != "" && gover.Compare(gover.Local(), goVersion) < 0 { + toolchain.SwitchOrFatal(ctx, &gover.TooNewError{ + What: "-go flag", + GoVersion: goVersion, + }) + } + + mg, err := modload.LoadModGraph(ctx, goVersion) + if err != nil { + base.Fatal(err) + } + + w := bufio.NewWriter(os.Stdout) + defer w.Flush() + + format := func(m module.Version) { + w.WriteString(m.Path) + if m.Version != "" { + w.WriteString("@") + w.WriteString(m.Version) + } + } + + mg.WalkBreadthFirst(func(m module.Version) { + reqs, _ := mg.RequiredBy(m) + for _, r := range reqs { + format(m) + w.WriteByte(' ') + format(r) + w.WriteByte('\n') + } + }) +} diff --git a/src/cmd/go/internal/modcmd/init.go b/src/cmd/go/internal/modcmd/init.go new file mode 100644 index 0000000..facdaa9 --- /dev/null +++ b/src/cmd/go/internal/modcmd/init.go @@ -0,0 +1,49 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// go mod init + +package modcmd + +import ( + "cmd/go/internal/base" + "cmd/go/internal/modload" + "context" +) + +var cmdInit = &base.Command{ + UsageLine: "go mod init [module-path]", + Short: "initialize new module in current directory", + Long: ` +Init initializes and writes a new go.mod file in the current directory, in +effect creating a new module rooted at the current directory. The go.mod file +must not already exist. + +Init accepts one optional argument, the module path for the new module. If the +module path argument is omitted, init will attempt to infer the module path +using import comments in .go files, vendoring tool configuration files (like +Gopkg.lock), and the current directory (if in GOPATH). + +See https://golang.org/ref/mod#go-mod-init for more about 'go mod init'. +`, + Run: runInit, +} + +func init() { + base.AddChdirFlag(&cmdInit.Flag) + base.AddModCommonFlags(&cmdInit.Flag) +} + +func runInit(ctx context.Context, cmd *base.Command, args []string) { + if len(args) > 1 { + base.Fatalf("go: 'go mod init' accepts at most one argument") + } + var modPath string + if len(args) == 1 { + modPath = args[0] + } + + modload.ForceUseModules = true + modload.CreateModFile(ctx, modPath) // does all the hard work +} diff --git a/src/cmd/go/internal/modcmd/mod.go b/src/cmd/go/internal/modcmd/mod.go new file mode 100644 index 0000000..125ba33 --- /dev/null +++ b/src/cmd/go/internal/modcmd/mod.go @@ -0,0 +1,33 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package modcmd implements the “go mod” command. +package modcmd + +import ( + "cmd/go/internal/base" +) + +var CmdMod = &base.Command{ + UsageLine: "go mod", + Short: "module maintenance", + Long: `Go mod provides access to operations on modules. + +Note that support for modules is built into all the go commands, +not just 'go mod'. For example, day-to-day adding, removing, upgrading, +and downgrading of dependencies should be done using 'go get'. +See 'go help modules' for an overview of module functionality. + `, + + Commands: []*base.Command{ + cmdDownload, + cmdEdit, + cmdGraph, + cmdInit, + cmdTidy, + cmdVendor, + cmdVerify, + cmdWhy, + }, +} diff --git a/src/cmd/go/internal/modcmd/tidy.go b/src/cmd/go/internal/modcmd/tidy.go new file mode 100644 index 0000000..36be926 --- /dev/null +++ b/src/cmd/go/internal/modcmd/tidy.go @@ -0,0 +1,139 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// go mod tidy + +package modcmd + +import ( + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/gover" + "cmd/go/internal/imports" + "cmd/go/internal/modload" + "cmd/go/internal/toolchain" + "context" + "fmt" + + "golang.org/x/mod/modfile" +) + +var cmdTidy = &base.Command{ + UsageLine: "go mod tidy [-e] [-v] [-x] [-go=version] [-compat=version]", + Short: "add missing and remove unused modules", + Long: ` +Tidy makes sure go.mod matches the source code in the module. +It adds any missing modules necessary to build the current module's +packages and dependencies, and it removes unused modules that +don't provide any relevant packages. It also adds any missing entries +to go.sum and removes any unnecessary ones. + +The -v flag causes tidy to print information about removed modules +to standard error. + +The -e flag causes tidy to attempt to proceed despite errors +encountered while loading packages. + +The -go flag causes tidy to update the 'go' directive in the go.mod +file to the given version, which may change which module dependencies +are retained as explicit requirements in the go.mod file. +(Go versions 1.17 and higher retain more requirements in order to +support lazy module loading.) + +The -compat flag preserves any additional checksums needed for the +'go' command from the indicated major Go release to successfully load +the module graph, and causes tidy to error out if that version of the +'go' command would load any imported package from a different module +version. By default, tidy acts as if the -compat flag were set to the +version prior to the one indicated by the 'go' directive in the go.mod +file. + +The -x flag causes tidy to print the commands download executes. + +See https://golang.org/ref/mod#go-mod-tidy for more about 'go mod tidy'. + `, + Run: runTidy, +} + +var ( + tidyE bool // if true, report errors but proceed anyway. + tidyGo goVersionFlag // go version to write to the tidied go.mod file (toggles lazy loading) + tidyCompat goVersionFlag // go version for which the tidied go.mod and go.sum files should be “compatible” +) + +func init() { + cmdTidy.Flag.BoolVar(&cfg.BuildV, "v", false, "") + cmdTidy.Flag.BoolVar(&cfg.BuildX, "x", false, "") + cmdTidy.Flag.BoolVar(&tidyE, "e", false, "") + cmdTidy.Flag.Var(&tidyGo, "go", "") + cmdTidy.Flag.Var(&tidyCompat, "compat", "") + base.AddChdirFlag(&cmdTidy.Flag) + base.AddModCommonFlags(&cmdTidy.Flag) +} + +// A goVersionFlag is a flag.Value representing a supported Go version. +// +// (Note that the -go argument to 'go mod edit' is *not* a goVersionFlag. +// It intentionally allows newer-than-supported versions as arguments.) +type goVersionFlag struct { + v string +} + +func (f *goVersionFlag) String() string { return f.v } +func (f *goVersionFlag) Get() any { return f.v } + +func (f *goVersionFlag) Set(s string) error { + if s != "" { + latest := gover.Local() + if !modfile.GoVersionRE.MatchString(s) { + return fmt.Errorf("expecting a Go version like %q", latest) + } + if gover.Compare(s, latest) > 0 { + return fmt.Errorf("maximum supported Go version is %s", latest) + } + } + + f.v = s + return nil +} + +func runTidy(ctx context.Context, cmd *base.Command, args []string) { + if len(args) > 0 { + base.Fatalf("go: 'go mod tidy' accepts no arguments") + } + + // Tidy aims to make 'go test' reproducible for any package in 'all', so we + // need to include test dependencies. For modules that specify go 1.15 or + // earlier this is a no-op (because 'all' saturates transitive test + // dependencies). + // + // However, with lazy loading (go 1.16+) 'all' includes only the packages that + // are transitively imported by the main module, not the test dependencies of + // those packages. In order to make 'go test' reproducible for the packages + // that are in 'all' but outside of the main module, we must explicitly + // request that their test dependencies be included. + modload.ForceUseModules = true + modload.RootMode = modload.NeedRoot + + goVersion := tidyGo.String() + if goVersion != "" && gover.Compare(gover.Local(), goVersion) < 0 { + toolchain.SwitchOrFatal(ctx, &gover.TooNewError{ + What: "-go flag", + GoVersion: goVersion, + }) + } + + modload.LoadPackages(ctx, modload.PackageOpts{ + TidyGoVersion: tidyGo.String(), + Tags: imports.AnyTags(), + Tidy: true, + TidyCompatibleVersion: tidyCompat.String(), + VendorModulesInGOROOTSrc: true, + ResolveMissingImports: true, + LoadTests: true, + AllowErrors: tidyE, + SilenceMissingStdImports: true, + Switcher: new(toolchain.Switcher), + }, "all") +} diff --git a/src/cmd/go/internal/modcmd/vendor.go b/src/cmd/go/internal/modcmd/vendor.go new file mode 100644 index 0000000..3db85bd --- /dev/null +++ b/src/cmd/go/internal/modcmd/vendor.go @@ -0,0 +1,481 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modcmd + +import ( + "bytes" + "context" + "errors" + "fmt" + "go/build" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "sort" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/gover" + "cmd/go/internal/imports" + "cmd/go/internal/load" + "cmd/go/internal/modload" + "cmd/go/internal/str" + + "golang.org/x/mod/module" +) + +var cmdVendor = &base.Command{ + UsageLine: "go mod vendor [-e] [-v] [-o outdir]", + Short: "make vendored copy of dependencies", + Long: ` +Vendor resets the main module's vendor directory to include all packages +needed to build and test all the main module's packages. +It does not include test code for vendored packages. + +The -v flag causes vendor to print the names of vendored +modules and packages to standard error. + +The -e flag causes vendor to attempt to proceed despite errors +encountered while loading packages. + +The -o flag causes vendor to create the vendor directory at the given +path instead of "vendor". The go command can only use a vendor directory +named "vendor" within the module root directory, so this flag is +primarily useful for other tools. + +See https://golang.org/ref/mod#go-mod-vendor for more about 'go mod vendor'. + `, + Run: runVendor, +} + +var vendorE bool // if true, report errors but proceed anyway +var vendorO string // if set, overrides the default output directory + +func init() { + cmdVendor.Flag.BoolVar(&cfg.BuildV, "v", false, "") + cmdVendor.Flag.BoolVar(&vendorE, "e", false, "") + cmdVendor.Flag.StringVar(&vendorO, "o", "", "") + base.AddChdirFlag(&cmdVendor.Flag) + base.AddModCommonFlags(&cmdVendor.Flag) +} + +func runVendor(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + if modload.WorkFilePath() != "" { + base.Fatalf("go: 'go mod vendor' cannot be run in workspace mode. Run 'go work vendor' to vendor the workspace or set 'GOWORK=off' to exit workspace mode.") + } + RunVendor(ctx, vendorE, vendorO, args) +} + +func RunVendor(ctx context.Context, vendorE bool, vendorO string, args []string) { + if len(args) != 0 { + base.Fatalf("go: 'go mod vendor' accepts no arguments") + } + modload.ForceUseModules = true + modload.RootMode = modload.NeedRoot + + loadOpts := modload.PackageOpts{ + Tags: imports.AnyTags(), + VendorModulesInGOROOTSrc: true, + ResolveMissingImports: true, + UseVendorAll: true, + AllowErrors: vendorE, + SilenceMissingStdImports: true, + } + _, pkgs := modload.LoadPackages(ctx, loadOpts, "all") + + var vdir string + switch { + case filepath.IsAbs(vendorO): + vdir = vendorO + case vendorO != "": + vdir = filepath.Join(base.Cwd(), vendorO) + default: + vdir = filepath.Join(modload.VendorDir()) + } + if err := os.RemoveAll(vdir); err != nil { + base.Fatal(err) + } + + modpkgs := make(map[module.Version][]string) + for _, pkg := range pkgs { + m := modload.PackageModule(pkg) + if m.Path == "" || modload.MainModules.Contains(m.Path) { + continue + } + modpkgs[m] = append(modpkgs[m], pkg) + } + + includeAllReplacements := false + includeGoVersions := false + isExplicit := map[module.Version]bool{} + gv := modload.MainModules.GoVersion() + if gover.Compare(gv, "1.14") >= 0 && (modload.FindGoWork(base.Cwd()) != "" || modload.ModFile().Go != nil) { + // If the Go version is at least 1.14, annotate all explicit 'require' and + // 'replace' targets found in the go.mod file so that we can perform a + // stronger consistency check when -mod=vendor is set. + for _, m := range modload.MainModules.Versions() { + if modFile := modload.MainModules.ModFile(m); modFile != nil { + for _, r := range modFile.Require { + isExplicit[r.Mod] = true + } + } + + } + includeAllReplacements = true + } + if gover.Compare(gv, "1.17") >= 0 { + // If the Go version is at least 1.17, annotate all modules with their + // 'go' version directives. + includeGoVersions = true + } + + var vendorMods []module.Version + for m := range isExplicit { + vendorMods = append(vendorMods, m) + } + for m := range modpkgs { + if !isExplicit[m] { + vendorMods = append(vendorMods, m) + } + } + gover.ModSort(vendorMods) + + var ( + buf bytes.Buffer + w io.Writer = &buf + ) + if cfg.BuildV { + w = io.MultiWriter(&buf, os.Stderr) + } + + if modload.MainModules.WorkFile() != nil { + fmt.Fprintf(w, "## workspace\n") + } + + replacementWritten := make(map[module.Version]bool) + for _, m := range vendorMods { + replacement := modload.Replacement(m) + line := moduleLine(m, replacement) + replacementWritten[m] = true + io.WriteString(w, line) + + goVersion := "" + if includeGoVersions { + goVersion = modload.ModuleInfo(ctx, m.Path).GoVersion + } + switch { + case isExplicit[m] && goVersion != "": + fmt.Fprintf(w, "## explicit; go %s\n", goVersion) + case isExplicit[m]: + io.WriteString(w, "## explicit\n") + case goVersion != "": + fmt.Fprintf(w, "## go %s\n", goVersion) + } + + pkgs := modpkgs[m] + sort.Strings(pkgs) + for _, pkg := range pkgs { + fmt.Fprintf(w, "%s\n", pkg) + vendorPkg(vdir, pkg) + } + } + + if includeAllReplacements { + // Record unused and wildcard replacements at the end of the modules.txt file: + // without access to the complete build list, the consumer of the vendor + // directory can't otherwise determine that those replacements had no effect. + for _, m := range modload.MainModules.Versions() { + if workFile := modload.MainModules.WorkFile(); workFile != nil { + for _, r := range workFile.Replace { + if replacementWritten[r.Old] { + // We already recorded this replacement. + continue + } + replacementWritten[r.Old] = true + + line := moduleLine(r.Old, r.New) + buf.WriteString(line) + if cfg.BuildV { + os.Stderr.WriteString(line) + } + } + } + if modFile := modload.MainModules.ModFile(m); modFile != nil { + for _, r := range modFile.Replace { + if replacementWritten[r.Old] { + // We already recorded this replacement. + continue + } + replacementWritten[r.Old] = true + rNew := modload.Replacement(r.Old) + if rNew == (module.Version{}) { + // There is no replacement. Don't try to write it. + continue + } + + line := moduleLine(r.Old, rNew) + buf.WriteString(line) + if cfg.BuildV { + os.Stderr.WriteString(line) + } + } + } + } + } + + if buf.Len() == 0 { + fmt.Fprintf(os.Stderr, "go: no dependencies to vendor\n") + return + } + + if err := os.MkdirAll(vdir, 0777); err != nil { + base.Fatal(err) + } + + if err := os.WriteFile(filepath.Join(vdir, "modules.txt"), buf.Bytes(), 0666); err != nil { + base.Fatal(err) + } +} + +func moduleLine(m, r module.Version) string { + b := new(strings.Builder) + b.WriteString("# ") + b.WriteString(m.Path) + if m.Version != "" { + b.WriteString(" ") + b.WriteString(m.Version) + } + if r.Path != "" { + if str.HasFilePathPrefix(filepath.Clean(r.Path), "vendor") { + base.Fatalf("go: replacement path %s inside vendor directory", r.Path) + } + b.WriteString(" => ") + b.WriteString(r.Path) + if r.Version != "" { + b.WriteString(" ") + b.WriteString(r.Version) + } + } + b.WriteString("\n") + return b.String() +} + +func vendorPkg(vdir, pkg string) { + src, realPath, _ := modload.Lookup("", false, pkg) + if src == "" { + base.Errorf("internal error: no pkg for %s\n", pkg) + return + } + if realPath != pkg { + // TODO(#26904): Revisit whether this behavior still makes sense. + // This should actually be impossible today, because the import map is the + // identity function for packages outside of the standard library. + // + // Part of the purpose of the vendor directory is to allow the packages in + // the module to continue to build in GOPATH mode, and GOPATH-mode users + // won't know about replacement aliasing. How important is it to maintain + // compatibility? + fmt.Fprintf(os.Stderr, "warning: %s imported as both %s and %s; making two copies.\n", realPath, realPath, pkg) + } + + copiedFiles := make(map[string]bool) + dst := filepath.Join(vdir, pkg) + copyDir(dst, src, matchPotentialSourceFile, copiedFiles) + if m := modload.PackageModule(realPath); m.Path != "" { + copyMetadata(m.Path, realPath, dst, src, copiedFiles) + } + + ctx := build.Default + ctx.UseAllFiles = true + bp, err := ctx.ImportDir(src, build.IgnoreVendor) + // Because UseAllFiles is set on the build.Context, it's possible ta get + // a MultiplePackageError on an otherwise valid package: the package could + // have different names for GOOS=windows and GOOS=mac for example. On the + // other hand if there's a NoGoError, the package might have source files + // specifying "//go:build ignore" those packages should be skipped because + // embeds from ignored files can't be used. + // TODO(#42504): Find a better way to avoid errors from ImportDir. We'll + // need to figure this out when we switch to PackagesAndErrors as per the + // TODO above. + var multiplePackageError *build.MultiplePackageError + var noGoError *build.NoGoError + if err != nil { + if errors.As(err, &noGoError) { + return // No source files in this package are built. Skip embeds in ignored files. + } else if !errors.As(err, &multiplePackageError) { // multiplePackageErrors are OK, but others are not. + base.Fatalf("internal error: failed to find embedded files of %s: %v\n", pkg, err) + } + } + var embedPatterns []string + if gover.Compare(modload.MainModules.GoVersion(), "1.22") >= 0 { + embedPatterns = bp.EmbedPatterns + } else { + // Maintain the behavior of https://github.com/golang/go/issues/63473 + // so that we continue to agree with older versions of the go command + // about the contents of vendor directories in existing modules + embedPatterns = str.StringList(bp.EmbedPatterns, bp.TestEmbedPatterns, bp.XTestEmbedPatterns) + } + embeds, err := load.ResolveEmbed(bp.Dir, embedPatterns) + if err != nil { + base.Fatal(err) + } + for _, embed := range embeds { + embedDst := filepath.Join(dst, embed) + if copiedFiles[embedDst] { + continue + } + + // Copy the file as is done by copyDir below. + r, err := os.Open(filepath.Join(src, embed)) + if err != nil { + base.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(embedDst), 0777); err != nil { + base.Fatal(err) + } + w, err := os.Create(embedDst) + if err != nil { + base.Fatal(err) + } + if _, err := io.Copy(w, r); err != nil { + base.Fatal(err) + } + r.Close() + if err := w.Close(); err != nil { + base.Fatal(err) + } + } +} + +type metakey struct { + modPath string + dst string +} + +var copiedMetadata = make(map[metakey]bool) + +// copyMetadata copies metadata files from parents of src to parents of dst, +// stopping after processing the src parent for modPath. +func copyMetadata(modPath, pkg, dst, src string, copiedFiles map[string]bool) { + for parent := 0; ; parent++ { + if copiedMetadata[metakey{modPath, dst}] { + break + } + copiedMetadata[metakey{modPath, dst}] = true + if parent > 0 { + copyDir(dst, src, matchMetadata, copiedFiles) + } + if modPath == pkg { + break + } + pkg = path.Dir(pkg) + dst = filepath.Dir(dst) + src = filepath.Dir(src) + } +} + +// metaPrefixes is the list of metadata file prefixes. +// Vendoring copies metadata files from parents of copied directories. +// Note that this list could be arbitrarily extended, and it is longer +// in other tools (such as godep or dep). By using this limited set of +// prefixes and also insisting on capitalized file names, we are trying +// to nudge people toward more agreement on the naming +// and also trying to avoid false positives. +var metaPrefixes = []string{ + "AUTHORS", + "CONTRIBUTORS", + "COPYLEFT", + "COPYING", + "COPYRIGHT", + "LEGAL", + "LICENSE", + "NOTICE", + "PATENTS", +} + +// matchMetadata reports whether info is a metadata file. +func matchMetadata(dir string, info fs.DirEntry) bool { + name := info.Name() + for _, p := range metaPrefixes { + if strings.HasPrefix(name, p) { + return true + } + } + return false +} + +// matchPotentialSourceFile reports whether info may be relevant to a build operation. +func matchPotentialSourceFile(dir string, info fs.DirEntry) bool { + if strings.HasSuffix(info.Name(), "_test.go") { + return false + } + if info.Name() == "go.mod" || info.Name() == "go.sum" { + if gv := modload.MainModules.GoVersion(); gover.Compare(gv, "1.17") >= 0 { + // As of Go 1.17, we strip go.mod and go.sum files from dependency modules. + // Otherwise, 'go' commands invoked within the vendor subtree may misidentify + // an arbitrary directory within the vendor tree as a module root. + // (See https://golang.org/issue/42970.) + return false + } + } + if strings.HasSuffix(info.Name(), ".go") { + f, err := fsys.Open(filepath.Join(dir, info.Name())) + if err != nil { + base.Fatal(err) + } + defer f.Close() + + content, err := imports.ReadImports(f, false, nil) + if err == nil && !imports.ShouldBuild(content, imports.AnyTags()) { + // The file is explicitly tagged "ignore", so it can't affect the build. + // Leave it out. + return false + } + return true + } + + // We don't know anything about this file, so optimistically assume that it is + // needed. + return true +} + +// copyDir copies all regular files satisfying match(info) from src to dst. +func copyDir(dst, src string, match func(dir string, info fs.DirEntry) bool, copiedFiles map[string]bool) { + files, err := os.ReadDir(src) + if err != nil { + base.Fatal(err) + } + if err := os.MkdirAll(dst, 0777); err != nil { + base.Fatal(err) + } + for _, file := range files { + if file.IsDir() || !file.Type().IsRegular() || !match(src, file) { + continue + } + copiedFiles[file.Name()] = true + r, err := os.Open(filepath.Join(src, file.Name())) + if err != nil { + base.Fatal(err) + } + dstPath := filepath.Join(dst, file.Name()) + copiedFiles[dstPath] = true + w, err := os.Create(dstPath) + if err != nil { + base.Fatal(err) + } + if _, err := io.Copy(w, r); err != nil { + base.Fatal(err) + } + r.Close() + if err := w.Close(); err != nil { + base.Fatal(err) + } + } +} diff --git a/src/cmd/go/internal/modcmd/verify.go b/src/cmd/go/internal/modcmd/verify.go new file mode 100644 index 0000000..d07f730 --- /dev/null +++ b/src/cmd/go/internal/modcmd/verify.go @@ -0,0 +1,143 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modcmd + +import ( + "bytes" + "context" + "errors" + "fmt" + "io/fs" + "os" + "runtime" + + "cmd/go/internal/base" + "cmd/go/internal/gover" + "cmd/go/internal/modfetch" + "cmd/go/internal/modload" + + "golang.org/x/mod/module" + "golang.org/x/mod/sumdb/dirhash" +) + +var cmdVerify = &base.Command{ + UsageLine: "go mod verify", + Short: "verify dependencies have expected content", + Long: ` +Verify checks that the dependencies of the current module, +which are stored in a local downloaded source cache, have not been +modified since being downloaded. If all the modules are unmodified, +verify prints "all modules verified." Otherwise it reports which +modules have been changed and causes 'go mod' to exit with a +non-zero status. + +See https://golang.org/ref/mod#go-mod-verify for more about 'go mod verify'. + `, + Run: runVerify, +} + +func init() { + base.AddChdirFlag(&cmdVerify.Flag) + base.AddModCommonFlags(&cmdVerify.Flag) +} + +func runVerify(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + + if len(args) != 0 { + // NOTE(rsc): Could take a module pattern. + base.Fatalf("go: verify takes no arguments") + } + modload.ForceUseModules = true + modload.RootMode = modload.NeedRoot + + // Only verify up to GOMAXPROCS zips at once. + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + + mg, err := modload.LoadModGraph(ctx, "") + if err != nil { + base.Fatal(err) + } + mods := mg.BuildList() + // Use a slice of result channels, so that the output is deterministic. + errsChans := make([]<-chan []error, len(mods)) + + for i, mod := range mods { + sem <- token{} + errsc := make(chan []error, 1) + errsChans[i] = errsc + mod := mod // use a copy to avoid data races + go func() { + errsc <- verifyMod(ctx, mod) + <-sem + }() + } + + ok := true + for _, errsc := range errsChans { + errs := <-errsc + for _, err := range errs { + base.Errorf("%s", err) + ok = false + } + } + if ok { + fmt.Printf("all modules verified\n") + } +} + +func verifyMod(ctx context.Context, mod module.Version) []error { + if gover.IsToolchain(mod.Path) { + // "go" and "toolchain" have no disk footprint; nothing to verify. + return nil + } + if modload.MainModules.Contains(mod.Path) { + return nil + } + var errs []error + zip, zipErr := modfetch.CachePath(ctx, mod, "zip") + if zipErr == nil { + _, zipErr = os.Stat(zip) + } + dir, dirErr := modfetch.DownloadDir(ctx, mod) + data, err := os.ReadFile(zip + "hash") + if err != nil { + if zipErr != nil && errors.Is(zipErr, fs.ErrNotExist) && + dirErr != nil && errors.Is(dirErr, fs.ErrNotExist) { + // Nothing downloaded yet. Nothing to verify. + return nil + } + errs = append(errs, fmt.Errorf("%s %s: missing ziphash: %v", mod.Path, mod.Version, err)) + return errs + } + h := string(bytes.TrimSpace(data)) + + if zipErr != nil && errors.Is(zipErr, fs.ErrNotExist) { + // ok + } else { + hZ, err := dirhash.HashZip(zip, dirhash.DefaultHash) + if err != nil { + errs = append(errs, fmt.Errorf("%s %s: %v", mod.Path, mod.Version, err)) + return errs + } else if hZ != h { + errs = append(errs, fmt.Errorf("%s %s: zip has been modified (%v)", mod.Path, mod.Version, zip)) + } + } + if dirErr != nil && errors.Is(dirErr, fs.ErrNotExist) { + // ok + } else { + hD, err := dirhash.HashDir(dir, mod.Path+"@"+mod.Version, dirhash.DefaultHash) + if err != nil { + + errs = append(errs, fmt.Errorf("%s %s: %v", mod.Path, mod.Version, err)) + return errs + } + if hD != h { + errs = append(errs, fmt.Errorf("%s %s: dir has been modified (%v)", mod.Path, mod.Version, dir)) + } + } + return errs +} diff --git a/src/cmd/go/internal/modcmd/why.go b/src/cmd/go/internal/modcmd/why.go new file mode 100644 index 0000000..198672d --- /dev/null +++ b/src/cmd/go/internal/modcmd/why.go @@ -0,0 +1,143 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modcmd + +import ( + "context" + "fmt" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/imports" + "cmd/go/internal/modload" +) + +var cmdWhy = &base.Command{ + UsageLine: "go mod why [-m] [-vendor] packages...", + Short: "explain why packages or modules are needed", + Long: ` +Why shows a shortest path in the import graph from the main module to +each of the listed packages. If the -m flag is given, why treats the +arguments as a list of modules and finds a path to any package in each +of the modules. + +By default, why queries the graph of packages matched by "go list all", +which includes tests for reachable packages. The -vendor flag causes why +to exclude tests of dependencies. + +The output is a sequence of stanzas, one for each package or module +name on the command line, separated by blank lines. Each stanza begins +with a comment line "# package" or "# module" giving the target +package or module. Subsequent lines give a path through the import +graph, one package per line. If the package or module is not +referenced from the main module, the stanza will display a single +parenthesized note indicating that fact. + +For example: + + $ go mod why golang.org/x/text/language golang.org/x/text/encoding + # golang.org/x/text/language + rsc.io/quote + rsc.io/sampler + golang.org/x/text/language + + # golang.org/x/text/encoding + (main module does not need package golang.org/x/text/encoding) + $ + +See https://golang.org/ref/mod#go-mod-why for more about 'go mod why'. + `, +} + +var ( + whyM = cmdWhy.Flag.Bool("m", false, "") + whyVendor = cmdWhy.Flag.Bool("vendor", false, "") +) + +func init() { + cmdWhy.Run = runWhy // break init cycle + base.AddChdirFlag(&cmdWhy.Flag) + base.AddModCommonFlags(&cmdWhy.Flag) +} + +func runWhy(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + modload.ForceUseModules = true + modload.RootMode = modload.NeedRoot + modload.ExplicitWriteGoMod = true // don't write go.mod in ListModules + + loadOpts := modload.PackageOpts{ + Tags: imports.AnyTags(), + VendorModulesInGOROOTSrc: true, + LoadTests: !*whyVendor, + SilencePackageErrors: true, + UseVendorAll: *whyVendor, + } + + if *whyM { + for _, arg := range args { + if strings.Contains(arg, "@") { + base.Fatalf("go: %s: 'go mod why' requires a module path, not a version query", arg) + } + } + + mods, err := modload.ListModules(ctx, args, 0, "") + if err != nil { + base.Fatal(err) + } + + byModule := make(map[string][]string) + _, pkgs := modload.LoadPackages(ctx, loadOpts, "all") + for _, path := range pkgs { + m := modload.PackageModule(path) + if m.Path != "" { + byModule[m.Path] = append(byModule[m.Path], path) + } + } + sep := "" + for _, m := range mods { + best := "" + bestDepth := 1000000000 + for _, path := range byModule[m.Path] { + d := modload.WhyDepth(path) + if d > 0 && d < bestDepth { + best = path + bestDepth = d + } + } + why := modload.Why(best) + if why == "" { + vendoring := "" + if *whyVendor { + vendoring = " to vendor" + } + why = "(main module does not need" + vendoring + " module " + m.Path + ")\n" + } + fmt.Printf("%s# %s\n%s", sep, m.Path, why) + sep = "\n" + } + } else { + // Resolve to packages. + matches, _ := modload.LoadPackages(ctx, loadOpts, args...) + + modload.LoadPackages(ctx, loadOpts, "all") // rebuild graph, from main module (not from named packages) + + sep := "" + for _, m := range matches { + for _, path := range m.Pkgs { + why := modload.Why(path) + if why == "" { + vendoring := "" + if *whyVendor { + vendoring = " to vendor" + } + why = "(main module does not need" + vendoring + " package " + path + ")\n" + } + fmt.Printf("%s# %s\n%s", sep, path, why) + sep = "\n" + } + } + } +} |