Source file src/cmd/go/internal/modload/load.go

Documentation: cmd/go/internal/modload

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modload
     6  
     7  // This file contains the module-mode package loader, as well as some accessory
     8  // functions pertaining to the package import graph.
     9  //
    10  // There are two exported entry points into package loading — LoadPackages and
    11  // ImportFromFiles — both implemented in terms of loadFromRoots, which itself
    12  // manipulates an instance of the loader struct.
    13  //
    14  // Although most of the loading state is maintained in the loader struct,
    15  // one key piece - the build list - is a global, so that it can be modified
    16  // separate from the loading operation, such as during "go get"
    17  // upgrades/downgrades or in "go mod" operations.
    18  // TODO(#40775): It might be nice to make the loader take and return
    19  // a buildList rather than hard-coding use of the global.
    20  //
    21  // Loading is an iterative process. On each iteration, we try to load the
    22  // requested packages and their transitive imports, then try to resolve modules
    23  // for any imported packages that are still missing.
    24  //
    25  // The first step of each iteration identifies a set of “root” packages.
    26  // Normally the root packages are exactly those matching the named pattern
    27  // arguments. However, for the "all" meta-pattern, the final set of packages is
    28  // computed from the package import graph, and therefore cannot be an initial
    29  // input to loading that graph. Instead, the root packages for the "all" pattern
    30  // are those contained in the main module, and allPatternIsRoot parameter to the
    31  // loader instructs it to dynamically expand those roots to the full "all"
    32  // pattern as loading progresses.
    33  //
    34  // The pkgInAll flag on each loadPkg instance tracks whether that
    35  // package is known to match the "all" meta-pattern.
    36  // A package matches the "all" pattern if:
    37  // 	- it is in the main module, or
    38  // 	- it is imported by any test in the main module, or
    39  // 	- it is imported by another package in "all", or
    40  // 	- the main module specifies a go version ≤ 1.15, and the package is imported
    41  // 	  by a *test of* another package in "all".
    42  //
    43  // When we implement lazy loading, we will record the modules providing packages
    44  // in "all" even when we are only loading individual packages, so we set the
    45  // pkgInAll flag regardless of the whether the "all" pattern is a root.
    46  // (This is necessary to maintain the “import invariant” described in
    47  // https://golang.org/design/36460-lazy-module-loading.)
    48  //
    49  // Because "go mod vendor" prunes out the tests of vendored packages, the
    50  // behavior of the "all" pattern with -mod=vendor in Go 1.11–1.15 is the same
    51  // as the "all" pattern (regardless of the -mod flag) in 1.16+.
    52  // The loader uses the GoVersion parameter to determine whether the "all"
    53  // pattern should close over tests (as in Go 1.11–1.15) or stop at only those
    54  // packages transitively imported by the packages and tests in the main module
    55  // ("all" in Go 1.16+ and "go mod vendor" in Go 1.11+).
    56  //
    57  // Note that it is possible for a loaded package NOT to be in "all" even when we
    58  // are loading the "all" pattern. For example, packages that are transitive
    59  // dependencies of other roots named on the command line must be loaded, but are
    60  // not in "all". (The mod_notall test illustrates this behavior.)
    61  // Similarly, if the LoadTests flag is set but the "all" pattern does not close
    62  // over test dependencies, then when we load the test of a package that is in
    63  // "all" but outside the main module, the dependencies of that test will not
    64  // necessarily themselves be in "all". (That configuration does not arise in Go
    65  // 1.11–1.15, but it will be possible in Go 1.16+.)
    66  //
    67  // Loading proceeds from the roots, using a parallel work-queue with a limit on
    68  // the amount of active work (to avoid saturating disks, CPU cores, and/or
    69  // network connections). Each package is added to the queue the first time it is
    70  // imported by another package. When we have finished identifying the imports of
    71  // a package, we add the test for that package if it is needed. A test may be
    72  // needed if:
    73  // 	- the package matches a root pattern and tests of the roots were requested, or
    74  // 	- the package is in the main module and the "all" pattern is requested
    75  // 	  (because the "all" pattern includes the dependencies of tests in the main
    76  // 	  module), or
    77  // 	- the package is in "all" and the definition of "all" we are using includes
    78  // 	  dependencies of tests (as is the case in Go ≤1.15).
    79  //
    80  // After all available packages have been loaded, we examine the results to
    81  // identify any requested or imported packages that are still missing, and if
    82  // so, which modules we could add to the module graph in order to make the
    83  // missing packages available. We add those to the module graph and iterate,
    84  // until either all packages resolve successfully or we cannot identify any
    85  // module that would resolve any remaining missing package.
    86  //
    87  // If the main module is “tidy” (that is, if "go mod tidy" is a no-op for it)
    88  // and all requested packages are in "all", then loading completes in a single
    89  // iteration.
    90  // TODO(bcmills): We should also be able to load in a single iteration if the
    91  // requested packages all come from modules that are themselves tidy, regardless
    92  // of whether those packages are in "all". Today, that requires two iterations
    93  // if those packages are not found in existing dependencies of the main module.
    94  
    95  import (
    96  	"bytes"
    97  	"context"
    98  	"errors"
    99  	"fmt"
   100  	"go/build"
   101  	"io/fs"
   102  	"os"
   103  	"path"
   104  	pathpkg "path"
   105  	"path/filepath"
   106  	"reflect"
   107  	"runtime"
   108  	"sort"
   109  	"strings"
   110  	"sync"
   111  	"sync/atomic"
   112  
   113  	"cmd/go/internal/base"
   114  	"cmd/go/internal/cfg"
   115  	"cmd/go/internal/fsys"
   116  	"cmd/go/internal/imports"
   117  	"cmd/go/internal/modfetch"
   118  	"cmd/go/internal/mvs"
   119  	"cmd/go/internal/par"
   120  	"cmd/go/internal/search"
   121  	"cmd/go/internal/str"
   122  
   123  	"golang.org/x/mod/module"
   124  	"golang.org/x/mod/semver"
   125  )
   126  
   127  // loaded is the most recently-used package loader.
   128  // It holds details about individual packages.
   129  //
   130  // This variable should only be accessed directly in top-level exported
   131  // functions. All other functions that require or produce a *loader should pass
   132  // or return it as an explicit parameter.
   133  var loaded *loader
   134  
   135  // PackageOpts control the behavior of the LoadPackages function.
   136  type PackageOpts struct {
   137  	// GoVersion is the Go version to which the go.mod file should be updated
   138  	// after packages have been loaded.
   139  	//
   140  	// An empty GoVersion means to use the Go version already specified in the
   141  	// main module's go.mod file, or the latest Go version if there is no main
   142  	// module.
   143  	GoVersion string
   144  
   145  	// Tags are the build tags in effect (as interpreted by the
   146  	// cmd/go/internal/imports package).
   147  	// If nil, treated as equivalent to imports.Tags().
   148  	Tags map[string]bool
   149  
   150  	// Tidy, if true, requests that the build list and go.sum file be reduced to
   151  	// the minimial dependencies needed to reproducibly reload the requested
   152  	// packages.
   153  	Tidy bool
   154  
   155  	// TidyCompatibleVersion is the oldest Go version that must be able to
   156  	// reproducibly reload the requested packages.
   157  	//
   158  	// If empty, the compatible version is the Go version immediately prior to the
   159  	// 'go' version listed in the go.mod file.
   160  	TidyCompatibleVersion string
   161  
   162  	// VendorModulesInGOROOTSrc indicates that if we are within a module in
   163  	// GOROOT/src, packages in the module's vendor directory should be resolved as
   164  	// actual module dependencies (instead of standard-library packages).
   165  	VendorModulesInGOROOTSrc bool
   166  
   167  	// ResolveMissingImports indicates that we should attempt to add module
   168  	// dependencies as needed to resolve imports of packages that are not found.
   169  	//
   170  	// For commands that support the -mod flag, resolving imports may still fail
   171  	// if the flag is set to "readonly" (the default) or "vendor".
   172  	ResolveMissingImports bool
   173  
   174  	// AssumeRootsImported indicates that the transitive dependencies of the root
   175  	// packages should be treated as if those roots will be imported by the main
   176  	// module.
   177  	AssumeRootsImported bool
   178  
   179  	// AllowPackage, if non-nil, is called after identifying the module providing
   180  	// each package. If AllowPackage returns a non-nil error, that error is set
   181  	// for the package, and the imports and test of that package will not be
   182  	// loaded.
   183  	//
   184  	// AllowPackage may be invoked concurrently by multiple goroutines,
   185  	// and may be invoked multiple times for a given package path.
   186  	AllowPackage func(ctx context.Context, path string, mod module.Version) error
   187  
   188  	// LoadTests loads the test dependencies of each package matching a requested
   189  	// pattern. If ResolveMissingImports is also true, test dependencies will be
   190  	// resolved if missing.
   191  	LoadTests bool
   192  
   193  	// UseVendorAll causes the "all" package pattern to be interpreted as if
   194  	// running "go mod vendor" (or building with "-mod=vendor").
   195  	//
   196  	// This is a no-op for modules that declare 'go 1.16' or higher, for which this
   197  	// is the default (and only) interpretation of the "all" pattern in module mode.
   198  	UseVendorAll bool
   199  
   200  	// AllowErrors indicates that LoadPackages should not terminate the process if
   201  	// an error occurs.
   202  	AllowErrors bool
   203  
   204  	// SilencePackageErrors indicates that LoadPackages should not print errors
   205  	// that occur while matching or loading packages, and should not terminate the
   206  	// process if such an error occurs.
   207  	//
   208  	// Errors encountered in the module graph will still be reported.
   209  	//
   210  	// The caller may retrieve the silenced package errors using the Lookup
   211  	// function, and matching errors are still populated in the Errs field of the
   212  	// associated search.Match.)
   213  	SilencePackageErrors bool
   214  
   215  	// SilenceMissingStdImports indicates that LoadPackages should not print
   216  	// errors or terminate the process if an imported package is missing, and the
   217  	// import path looks like it might be in the standard library (perhaps in a
   218  	// future version).
   219  	SilenceMissingStdImports bool
   220  
   221  	// SilenceNoGoErrors indicates that LoadPackages should not print
   222  	// imports.ErrNoGo errors.
   223  	// This allows the caller to invoke LoadPackages (and report other errors)
   224  	// without knowing whether the requested packages exist for the given tags.
   225  	//
   226  	// Note that if a requested package does not exist *at all*, it will fail
   227  	// during module resolution and the error will not be suppressed.
   228  	SilenceNoGoErrors bool
   229  
   230  	// SilenceUnmatchedWarnings suppresses the warnings normally emitted for
   231  	// patterns that did not match any packages.
   232  	SilenceUnmatchedWarnings bool
   233  }
   234  
   235  // LoadPackages identifies the set of packages matching the given patterns and
   236  // loads the packages in the import graph rooted at that set.
   237  func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (matches []*search.Match, loadedPackages []string) {
   238  	if opts.Tags == nil {
   239  		opts.Tags = imports.Tags()
   240  	}
   241  
   242  	patterns = search.CleanPatterns(patterns)
   243  	matches = make([]*search.Match, 0, len(patterns))
   244  	allPatternIsRoot := false
   245  	for _, pattern := range patterns {
   246  		matches = append(matches, search.NewMatch(pattern))
   247  		if pattern == "all" {
   248  			allPatternIsRoot = true
   249  		}
   250  	}
   251  
   252  	updateMatches := func(rs *Requirements, ld *loader) {
   253  		for _, m := range matches {
   254  			switch {
   255  			case m.IsLocal():
   256  				// Evaluate list of file system directories on first iteration.
   257  				if m.Dirs == nil {
   258  					matchLocalDirs(ctx, m, rs)
   259  				}
   260  
   261  				// Make a copy of the directory list and translate to import paths.
   262  				// Note that whether a directory corresponds to an import path
   263  				// changes as the build list is updated, and a directory can change
   264  				// from not being in the build list to being in it and back as
   265  				// the exact version of a particular module increases during
   266  				// the loader iterations.
   267  				m.Pkgs = m.Pkgs[:0]
   268  				for _, dir := range m.Dirs {
   269  					pkg, err := resolveLocalPackage(ctx, dir, rs)
   270  					if err != nil {
   271  						if !m.IsLiteral() && (err == errPkgIsBuiltin || err == errPkgIsGorootSrc) {
   272  							continue // Don't include "builtin" or GOROOT/src in wildcard patterns.
   273  						}
   274  
   275  						// If we're outside of a module, ensure that the failure mode
   276  						// indicates that.
   277  						ModRoot()
   278  
   279  						if ld != nil {
   280  							m.AddError(err)
   281  						}
   282  						continue
   283  					}
   284  					m.Pkgs = append(m.Pkgs, pkg)
   285  				}
   286  
   287  			case m.IsLiteral():
   288  				m.Pkgs = []string{m.Pattern()}
   289  
   290  			case strings.Contains(m.Pattern(), "..."):
   291  				m.Errs = m.Errs[:0]
   292  				mg, err := rs.Graph(ctx)
   293  				if err != nil {
   294  					// The module graph is (or may be) incomplete — perhaps we failed to
   295  					// load the requirements of some module. This is an error in matching
   296  					// the patterns to packages, because we may be missing some packages
   297  					// or we may erroneously match packages in the wrong versions of
   298  					// modules. However, for cases like 'go list -e', the error should not
   299  					// necessarily prevent us from loading the packages we could find.
   300  					m.Errs = append(m.Errs, err)
   301  				}
   302  				matchPackages(ctx, m, opts.Tags, includeStd, mg.BuildList())
   303  
   304  			case m.Pattern() == "all":
   305  				if ld == nil {
   306  					// The initial roots are the packages in the main module.
   307  					// loadFromRoots will expand that to "all".
   308  					m.Errs = m.Errs[:0]
   309  					matchPackages(ctx, m, opts.Tags, omitStd, []module.Version{Target})
   310  				} else {
   311  					// Starting with the packages in the main module,
   312  					// enumerate the full list of "all".
   313  					m.Pkgs = ld.computePatternAll()
   314  				}
   315  
   316  			case m.Pattern() == "std" || m.Pattern() == "cmd":
   317  				if m.Pkgs == nil {
   318  					m.MatchPackages() // Locate the packages within GOROOT/src.
   319  				}
   320  
   321  			default:
   322  				panic(fmt.Sprintf("internal error: modload missing case for pattern %s", m.Pattern()))
   323  			}
   324  		}
   325  	}
   326  
   327  	initialRS, _ := loadModFile(ctx) // Ignore needCommit — we're going to commit at the end regardless.
   328  
   329  	ld := loadFromRoots(ctx, loaderParams{
   330  		PackageOpts:  opts,
   331  		requirements: initialRS,
   332  
   333  		allPatternIsRoot: allPatternIsRoot,
   334  
   335  		listRoots: func(rs *Requirements) (roots []string) {
   336  			updateMatches(rs, nil)
   337  			for _, m := range matches {
   338  				roots = append(roots, m.Pkgs...)
   339  			}
   340  			return roots
   341  		},
   342  	})
   343  
   344  	// One last pass to finalize wildcards.
   345  	updateMatches(ld.requirements, ld)
   346  
   347  	// List errors in matching patterns (such as directory permission
   348  	// errors for wildcard patterns).
   349  	if !ld.SilencePackageErrors {
   350  		for _, match := range matches {
   351  			for _, err := range match.Errs {
   352  				ld.errorf("%v\n", err)
   353  			}
   354  		}
   355  	}
   356  	base.ExitIfErrors()
   357  
   358  	if !opts.SilenceUnmatchedWarnings {
   359  		search.WarnUnmatched(matches)
   360  	}
   361  
   362  	if opts.Tidy {
   363  		if cfg.BuildV {
   364  			mg, _ := ld.requirements.Graph(ctx)
   365  
   366  			for _, m := range initialRS.rootModules {
   367  				var unused bool
   368  				if ld.requirements.depth == eager {
   369  					// m is unused if it was dropped from the module graph entirely. If it
   370  					// was only demoted from direct to indirect, it may still be in use via
   371  					// a transitive import.
   372  					unused = mg.Selected(m.Path) == "none"
   373  				} else {
   374  					// m is unused if it was dropped from the roots. If it is still present
   375  					// as a transitive dependency, that transitive dependency is not needed
   376  					// by any package or test in the main module.
   377  					_, ok := ld.requirements.rootSelected(m.Path)
   378  					unused = !ok
   379  				}
   380  				if unused {
   381  					fmt.Fprintf(os.Stderr, "unused %s\n", m.Path)
   382  				}
   383  			}
   384  		}
   385  
   386  		keep := keepSums(ctx, ld, ld.requirements, loadedZipSumsOnly)
   387  		if compatDepth := modDepthFromGoVersion(ld.TidyCompatibleVersion); compatDepth != ld.requirements.depth {
   388  			compatRS := newRequirements(compatDepth, ld.requirements.rootModules, ld.requirements.direct)
   389  			ld.checkTidyCompatibility(ctx, compatRS)
   390  
   391  			for m := range keepSums(ctx, ld, compatRS, loadedZipSumsOnly) {
   392  				keep[m] = true
   393  			}
   394  		}
   395  
   396  		if allowWriteGoMod {
   397  			modfetch.TrimGoSum(keep)
   398  
   399  			// commitRequirements below will also call WriteGoSum, but the "keep" map
   400  			// we have here could be strictly larger: commitRequirements only commits
   401  			// loaded.requirements, but here we may have also loaded (and want to
   402  			// preserve checksums for) additional entities from compatRS, which are
   403  			// only needed for compatibility with ld.TidyCompatibleVersion.
   404  			modfetch.WriteGoSum(keep)
   405  		}
   406  	}
   407  
   408  	// Success! Update go.mod and go.sum (if needed) and return the results.
   409  	loaded = ld
   410  	commitRequirements(ctx, loaded.GoVersion, loaded.requirements)
   411  
   412  	for _, pkg := range ld.pkgs {
   413  		if !pkg.isTest() {
   414  			loadedPackages = append(loadedPackages, pkg.path)
   415  		}
   416  	}
   417  	sort.Strings(loadedPackages)
   418  	return matches, loadedPackages
   419  }
   420  
   421  // matchLocalDirs is like m.MatchDirs, but tries to avoid scanning directories
   422  // outside of the standard library and active modules.
   423  func matchLocalDirs(ctx context.Context, m *search.Match, rs *Requirements) {
   424  	if !m.IsLocal() {
   425  		panic(fmt.Sprintf("internal error: resolveLocalDirs on non-local pattern %s", m.Pattern()))
   426  	}
   427  
   428  	if i := strings.Index(m.Pattern(), "..."); i >= 0 {
   429  		// The pattern is local, but it is a wildcard. Its packages will
   430  		// only resolve to paths if they are inside of the standard
   431  		// library, the main module, or some dependency of the main
   432  		// module. Verify that before we walk the filesystem: a filesystem
   433  		// walk in a directory like /var or /etc can be very expensive!
   434  		dir := filepath.Dir(filepath.Clean(m.Pattern()[:i+3]))
   435  		absDir := dir
   436  		if !filepath.IsAbs(dir) {
   437  			absDir = filepath.Join(base.Cwd(), dir)
   438  		}
   439  		if search.InDir(absDir, cfg.GOROOTsrc) == "" && search.InDir(absDir, ModRoot()) == "" && pathInModuleCache(ctx, absDir, rs) == "" {
   440  			m.Dirs = []string{}
   441  			m.AddError(fmt.Errorf("directory prefix %s outside available modules", base.ShortPath(absDir)))
   442  			return
   443  		}
   444  	}
   445  
   446  	m.MatchDirs()
   447  }
   448  
   449  // resolveLocalPackage resolves a filesystem path to a package path.
   450  func resolveLocalPackage(ctx context.Context, dir string, rs *Requirements) (string, error) {
   451  	var absDir string
   452  	if filepath.IsAbs(dir) {
   453  		absDir = filepath.Clean(dir)
   454  	} else {
   455  		absDir = filepath.Join(base.Cwd(), dir)
   456  	}
   457  
   458  	bp, err := cfg.BuildContext.ImportDir(absDir, 0)
   459  	if err != nil && (bp == nil || len(bp.IgnoredGoFiles) == 0) {
   460  		// golang.org/issue/32917: We should resolve a relative path to a
   461  		// package path only if the relative path actually contains the code
   462  		// for that package.
   463  		//
   464  		// If the named directory does not exist or contains no Go files,
   465  		// the package does not exist.
   466  		// Other errors may affect package loading, but not resolution.
   467  		if _, err := fsys.Stat(absDir); err != nil {
   468  			if os.IsNotExist(err) {
   469  				// Canonicalize OS-specific errors to errDirectoryNotFound so that error
   470  				// messages will be easier for users to search for.
   471  				return "", &fs.PathError{Op: "stat", Path: absDir, Err: errDirectoryNotFound}
   472  			}
   473  			return "", err
   474  		}
   475  		if _, noGo := err.(*build.NoGoError); noGo {
   476  			// A directory that does not contain any Go source files — even ignored
   477  			// ones! — is not a Go package, and we can't resolve it to a package
   478  			// path because that path could plausibly be provided by some other
   479  			// module.
   480  			//
   481  			// Any other error indicates that the package “exists” (at least in the
   482  			// sense that it cannot exist in any other module), but has some other
   483  			// problem (such as a syntax error).
   484  			return "", err
   485  		}
   486  	}
   487  
   488  	if modRoot != "" && absDir == modRoot {
   489  		if absDir == cfg.GOROOTsrc {
   490  			return "", errPkgIsGorootSrc
   491  		}
   492  		return targetPrefix, nil
   493  	}
   494  
   495  	// Note: The checks for @ here are just to avoid misinterpreting
   496  	// the module cache directories (formerly GOPATH/src/mod/foo@v1.5.2/bar).
   497  	// It's not strictly necessary but helpful to keep the checks.
   498  	if modRoot != "" && strings.HasPrefix(absDir, modRoot+string(filepath.Separator)) && !strings.Contains(absDir[len(modRoot):], "@") {
   499  		suffix := filepath.ToSlash(absDir[len(modRoot):])
   500  		if strings.HasPrefix(suffix, "/vendor/") {
   501  			if cfg.BuildMod != "vendor" {
   502  				return "", fmt.Errorf("without -mod=vendor, directory %s has no package path", absDir)
   503  			}
   504  
   505  			readVendorList()
   506  			pkg := strings.TrimPrefix(suffix, "/vendor/")
   507  			if _, ok := vendorPkgModule[pkg]; !ok {
   508  				return "", fmt.Errorf("directory %s is not a package listed in vendor/modules.txt", absDir)
   509  			}
   510  			return pkg, nil
   511  		}
   512  
   513  		if targetPrefix == "" {
   514  			pkg := strings.TrimPrefix(suffix, "/")
   515  			if pkg == "builtin" {
   516  				// "builtin" is a pseudo-package with a real source file.
   517  				// It's not included in "std", so it shouldn't resolve from "."
   518  				// within module "std" either.
   519  				return "", errPkgIsBuiltin
   520  			}
   521  			return pkg, nil
   522  		}
   523  
   524  		pkg := targetPrefix + suffix
   525  		if _, ok, err := dirInModule(pkg, targetPrefix, modRoot, true); err != nil {
   526  			return "", err
   527  		} else if !ok {
   528  			return "", &PackageNotInModuleError{Mod: Target, Pattern: pkg}
   529  		}
   530  		return pkg, nil
   531  	}
   532  
   533  	if sub := search.InDir(absDir, cfg.GOROOTsrc); sub != "" && sub != "." && !strings.Contains(sub, "@") {
   534  		pkg := filepath.ToSlash(sub)
   535  		if pkg == "builtin" {
   536  			return "", errPkgIsBuiltin
   537  		}
   538  		return pkg, nil
   539  	}
   540  
   541  	pkg := pathInModuleCache(ctx, absDir, rs)
   542  	if pkg == "" {
   543  		return "", fmt.Errorf("directory %s outside available modules", base.ShortPath(absDir))
   544  	}
   545  	return pkg, nil
   546  }
   547  
   548  var (
   549  	errDirectoryNotFound = errors.New("directory not found")
   550  	errPkgIsGorootSrc    = errors.New("GOROOT/src is not an importable package")
   551  	errPkgIsBuiltin      = errors.New(`"builtin" is a pseudo-package, not an importable package`)
   552  )
   553  
   554  // pathInModuleCache returns the import path of the directory dir,
   555  // if dir is in the module cache copy of a module in our build list.
   556  func pathInModuleCache(ctx context.Context, dir string, rs *Requirements) string {
   557  	tryMod := func(m module.Version) (string, bool) {
   558  		var root string
   559  		var err error
   560  		if repl := Replacement(m); repl.Path != "" && repl.Version == "" {
   561  			root = repl.Path
   562  			if !filepath.IsAbs(root) {
   563  				root = filepath.Join(ModRoot(), root)
   564  			}
   565  		} else if repl.Path != "" {
   566  			root, err = modfetch.DownloadDir(repl)
   567  		} else {
   568  			root, err = modfetch.DownloadDir(m)
   569  		}
   570  		if err != nil {
   571  			return "", false
   572  		}
   573  
   574  		sub := search.InDir(dir, root)
   575  		if sub == "" {
   576  			return "", false
   577  		}
   578  		sub = filepath.ToSlash(sub)
   579  		if strings.Contains(sub, "/vendor/") || strings.HasPrefix(sub, "vendor/") || strings.Contains(sub, "@") {
   580  			return "", false
   581  		}
   582  
   583  		return path.Join(m.Path, filepath.ToSlash(sub)), true
   584  	}
   585  
   586  	if rs.depth == lazy {
   587  		for _, m := range rs.rootModules {
   588  			if v, _ := rs.rootSelected(m.Path); v != m.Version {
   589  				continue // m is a root, but we have a higher root for the same path.
   590  			}
   591  			if importPath, ok := tryMod(m); ok {
   592  				// checkMultiplePaths ensures that a module can be used for at most one
   593  				// requirement, so this must be it.
   594  				return importPath
   595  			}
   596  		}
   597  	}
   598  
   599  	// None of the roots contained dir, or we're in eager mode and want to load
   600  	// the full module graph more aggressively. Either way, check the full graph
   601  	// to see if the directory is a non-root dependency.
   602  	//
   603  	// If the roots are not consistent with the full module graph, the selected
   604  	// versions of root modules may differ from what we already checked above.
   605  	// Re-check those paths too.
   606  
   607  	mg, _ := rs.Graph(ctx)
   608  	var importPath string
   609  	for _, m := range mg.BuildList() {
   610  		var found bool
   611  		importPath, found = tryMod(m)
   612  		if found {
   613  			break
   614  		}
   615  	}
   616  	return importPath
   617  }
   618  
   619  // ImportFromFiles adds modules to the build list as needed
   620  // to satisfy the imports in the named Go source files.
   621  //
   622  // Errors in missing dependencies are silenced.
   623  //
   624  // TODO(bcmills): Silencing errors seems off. Take a closer look at this and
   625  // figure out what the error-reporting actually ought to be.
   626  func ImportFromFiles(ctx context.Context, gofiles []string) {
   627  	rs := LoadModFile(ctx)
   628  
   629  	tags := imports.Tags()
   630  	imports, testImports, err := imports.ScanFiles(gofiles, tags)
   631  	if err != nil {
   632  		base.Fatalf("go: %v", err)
   633  	}
   634  
   635  	loaded = loadFromRoots(ctx, loaderParams{
   636  		PackageOpts: PackageOpts{
   637  			Tags:                  tags,
   638  			ResolveMissingImports: true,
   639  			SilencePackageErrors:  true,
   640  		},
   641  		requirements: rs,
   642  		listRoots: func(*Requirements) (roots []string) {
   643  			roots = append(roots, imports...)
   644  			roots = append(roots, testImports...)
   645  			return roots
   646  		},
   647  	})
   648  	commitRequirements(ctx, loaded.GoVersion, loaded.requirements)
   649  }
   650  
   651  // DirImportPath returns the effective import path for dir,
   652  // provided it is within the main module, or else returns ".".
   653  func DirImportPath(ctx context.Context, dir string) string {
   654  	if !HasModRoot() {
   655  		return "."
   656  	}
   657  	LoadModFile(ctx) // Sets targetPrefix.
   658  
   659  	if !filepath.IsAbs(dir) {
   660  		dir = filepath.Join(base.Cwd(), dir)
   661  	} else {
   662  		dir = filepath.Clean(dir)
   663  	}
   664  
   665  	if dir == modRoot {
   666  		return targetPrefix
   667  	}
   668  	if strings.HasPrefix(dir, modRoot+string(filepath.Separator)) {
   669  		suffix := filepath.ToSlash(dir[len(modRoot):])
   670  		if strings.HasPrefix(suffix, "/vendor/") {
   671  			return strings.TrimPrefix(suffix, "/vendor/")
   672  		}
   673  		return targetPrefix + suffix
   674  	}
   675  	return "."
   676  }
   677  
   678  // ImportMap returns the actual package import path
   679  // for an import path found in source code.
   680  // If the given import path does not appear in the source code
   681  // for the packages that have been loaded, ImportMap returns the empty string.
   682  func ImportMap(path string) string {
   683  	pkg, ok := loaded.pkgCache.Get(path).(*loadPkg)
   684  	if !ok {
   685  		return ""
   686  	}
   687  	return pkg.path
   688  }
   689  
   690  // PackageDir returns the directory containing the source code
   691  // for the package named by the import path.
   692  func PackageDir(path string) string {
   693  	pkg, ok := loaded.pkgCache.Get(path).(*loadPkg)
   694  	if !ok {
   695  		return ""
   696  	}
   697  	return pkg.dir
   698  }
   699  
   700  // PackageModule returns the module providing the package named by the import path.
   701  func PackageModule(path string) module.Version {
   702  	pkg, ok := loaded.pkgCache.Get(path).(*loadPkg)
   703  	if !ok {
   704  		return module.Version{}
   705  	}
   706  	return pkg.mod
   707  }
   708  
   709  // Lookup returns the source directory, import path, and any loading error for
   710  // the package at path as imported from the package in parentDir.
   711  // Lookup requires that one of the Load functions in this package has already
   712  // been called.
   713  func Lookup(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) {
   714  	if path == "" {
   715  		panic("Lookup called with empty package path")
   716  	}
   717  
   718  	if parentIsStd {
   719  		path = loaded.stdVendor(parentPath, path)
   720  	}
   721  	pkg, ok := loaded.pkgCache.Get(path).(*loadPkg)
   722  	if !ok {
   723  		// The loader should have found all the relevant paths.
   724  		// There are a few exceptions, though:
   725  		//	- during go list without -test, the p.Resolve calls to process p.TestImports and p.XTestImports
   726  		//	  end up here to canonicalize the import paths.
   727  		//	- during any load, non-loaded packages like "unsafe" end up here.
   728  		//	- during any load, build-injected dependencies like "runtime/cgo" end up here.
   729  		//	- because we ignore appengine/* in the module loader,
   730  		//	  the dependencies of any actual appengine/* library end up here.
   731  		dir := findStandardImportPath(path)
   732  		if dir != "" {
   733  			return dir, path, nil
   734  		}
   735  		return "", "", errMissing
   736  	}
   737  	return pkg.dir, pkg.path, pkg.err
   738  }
   739  
   740  // A loader manages the process of loading information about
   741  // the required packages for a particular build,
   742  // checking that the packages are available in the module set,
   743  // and updating the module set if needed.
   744  type loader struct {
   745  	loaderParams
   746  
   747  	// allClosesOverTests indicates whether the "all" pattern includes
   748  	// dependencies of tests outside the main module (as in Go 1.11–1.15).
   749  	// (Otherwise — as in Go 1.16+ — the "all" pattern includes only the packages
   750  	// transitively *imported by* the packages and tests in the main module.)
   751  	allClosesOverTests bool
   752  
   753  	work *par.Queue
   754  
   755  	// reset on each iteration
   756  	roots    []*loadPkg
   757  	pkgCache *par.Cache // package path (string) → *loadPkg
   758  	pkgs     []*loadPkg // transitive closure of loaded packages and tests; populated in buildStacks
   759  }
   760  
   761  // loaderParams configure the packages loaded by, and the properties reported
   762  // by, a loader instance.
   763  type loaderParams struct {
   764  	PackageOpts
   765  	requirements *Requirements
   766  
   767  	allPatternIsRoot bool // Is the "all" pattern an additional root?
   768  
   769  	listRoots func(rs *Requirements) []string
   770  }
   771  
   772  func (ld *loader) reset() {
   773  	select {
   774  	case <-ld.work.Idle():
   775  	default:
   776  		panic("loader.reset when not idle")
   777  	}
   778  
   779  	ld.roots = nil
   780  	ld.pkgCache = new(par.Cache)
   781  	ld.pkgs = nil
   782  }
   783  
   784  // errorf reports an error via either os.Stderr or base.Errorf,
   785  // according to whether ld.AllowErrors is set.
   786  func (ld *loader) errorf(format string, args ...interface{}) {
   787  	if ld.AllowErrors {
   788  		fmt.Fprintf(os.Stderr, format, args...)
   789  	} else {
   790  		base.Errorf(format, args...)
   791  	}
   792  }
   793  
   794  // A loadPkg records information about a single loaded package.
   795  type loadPkg struct {
   796  	// Populated at construction time:
   797  	path   string // import path
   798  	testOf *loadPkg
   799  
   800  	// Populated at construction time and updated by (*loader).applyPkgFlags:
   801  	flags atomicLoadPkgFlags
   802  
   803  	// Populated by (*loader).load:
   804  	mod         module.Version // module providing package
   805  	dir         string         // directory containing source code
   806  	err         error          // error loading package
   807  	imports     []*loadPkg     // packages imported by this one
   808  	testImports []string       // test-only imports, saved for use by pkg.test.
   809  	inStd       bool
   810  
   811  	// Populated by (*loader).pkgTest:
   812  	testOnce sync.Once
   813  	test     *loadPkg
   814  
   815  	// Populated by postprocessing in (*loader).buildStacks:
   816  	stack *loadPkg // package importing this one in minimal import stack for this pkg
   817  }
   818  
   819  // loadPkgFlags is a set of flags tracking metadata about a package.
   820  type loadPkgFlags int8
   821  
   822  const (
   823  	// pkgInAll indicates that the package is in the "all" package pattern,
   824  	// regardless of whether we are loading the "all" package pattern.
   825  	//
   826  	// When the pkgInAll flag and pkgImportsLoaded flags are both set, the caller
   827  	// who set the last of those flags must propagate the pkgInAll marking to all
   828  	// of the imports of the marked package.
   829  	//
   830  	// A test is marked with pkgInAll if that test would promote the packages it
   831  	// imports to be in "all" (such as when the test is itself within the main
   832  	// module, or when ld.allClosesOverTests is true).
   833  	pkgInAll loadPkgFlags = 1 << iota
   834  
   835  	// pkgIsRoot indicates that the package matches one of the root package
   836  	// patterns requested by the caller.
   837  	//
   838  	// If LoadTests is set, then when pkgIsRoot and pkgImportsLoaded are both set,
   839  	// the caller who set the last of those flags must populate a test for the
   840  	// package (in the pkg.test field).
   841  	//
   842  	// If the "all" pattern is included as a root, then non-test packages in "all"
   843  	// are also roots (and must be marked pkgIsRoot).
   844  	pkgIsRoot
   845  
   846  	// pkgFromRoot indicates that the package is in the transitive closure of
   847  	// imports starting at the roots. (Note that every package marked as pkgIsRoot
   848  	// is also trivially marked pkgFromRoot.)
   849  	pkgFromRoot
   850  
   851  	// pkgImportsLoaded indicates that the imports and testImports fields of a
   852  	// loadPkg have been populated.
   853  	pkgImportsLoaded
   854  )
   855  
   856  // has reports whether all of the flags in cond are set in f.
   857  func (f loadPkgFlags) has(cond loadPkgFlags) bool {
   858  	return f&cond == cond
   859  }
   860  
   861  // An atomicLoadPkgFlags stores a loadPkgFlags for which individual flags can be
   862  // added atomically.
   863  type atomicLoadPkgFlags struct {
   864  	bits int32
   865  }
   866  
   867  // update sets the given flags in af (in addition to any flags already set).
   868  //
   869  // update returns the previous flag state so that the caller may determine which
   870  // flags were newly-set.
   871  func (af *atomicLoadPkgFlags) update(flags loadPkgFlags) (old loadPkgFlags) {
   872  	for {
   873  		old := atomic.LoadInt32(&af.bits)
   874  		new := old | int32(flags)
   875  		if new == old || atomic.CompareAndSwapInt32(&af.bits, old, new) {
   876  			return loadPkgFlags(old)
   877  		}
   878  	}
   879  }
   880  
   881  // has reports whether all of the flags in cond are set in af.
   882  func (af *atomicLoadPkgFlags) has(cond loadPkgFlags) bool {
   883  	return loadPkgFlags(atomic.LoadInt32(&af.bits))&cond == cond
   884  }
   885  
   886  // isTest reports whether pkg is a test of another package.
   887  func (pkg *loadPkg) isTest() bool {
   888  	return pkg.testOf != nil
   889  }
   890  
   891  // fromExternalModule reports whether pkg was loaded from a module other than
   892  // the main module.
   893  func (pkg *loadPkg) fromExternalModule() bool {
   894  	if pkg.mod.Path == "" {
   895  		return false // loaded from the standard library, not a module
   896  	}
   897  	if pkg.mod.Path == Target.Path {
   898  		return false // loaded from the main module.
   899  	}
   900  	return true
   901  }
   902  
   903  var errMissing = errors.New("cannot find package")
   904  
   905  // loadFromRoots attempts to load the build graph needed to process a set of
   906  // root packages and their dependencies.
   907  //
   908  // The set of root packages is returned by the params.listRoots function, and
   909  // expanded to the full set of packages by tracing imports (and possibly tests)
   910  // as needed.
   911  func loadFromRoots(ctx context.Context, params loaderParams) *loader {
   912  	ld := &loader{
   913  		loaderParams: params,
   914  		work:         par.NewQueue(runtime.GOMAXPROCS(0)),
   915  	}
   916  
   917  	if ld.GoVersion == "" {
   918  		ld.GoVersion = modFileGoVersion()
   919  
   920  		if ld.Tidy && semver.Compare("v"+ld.GoVersion, "v"+LatestGoVersion()) > 0 {
   921  			ld.errorf("go mod tidy: go.mod file indicates go %s, but maximum supported version is %s\n", ld.GoVersion, LatestGoVersion())
   922  			base.ExitIfErrors()
   923  		}
   924  	}
   925  
   926  	if ld.Tidy {
   927  		if ld.TidyCompatibleVersion == "" {
   928  			ld.TidyCompatibleVersion = priorGoVersion(ld.GoVersion)
   929  		} else if semver.Compare("v"+ld.TidyCompatibleVersion, "v"+ld.GoVersion) > 0 {
   930  			// Each version of the Go toolchain knows how to interpret go.mod and
   931  			// go.sum files produced by all previous versions, so a compatibility
   932  			// version higher than the go.mod version adds nothing.
   933  			ld.TidyCompatibleVersion = ld.GoVersion
   934  		}
   935  	}
   936  
   937  	if semver.Compare("v"+ld.GoVersion, narrowAllVersionV) < 0 && !ld.UseVendorAll {
   938  		// The module's go version explicitly predates the change in "all" for lazy
   939  		// loading, so continue to use the older interpretation.
   940  		ld.allClosesOverTests = true
   941  	}
   942  
   943  	var err error
   944  	ld.requirements, err = convertDepth(ctx, ld.requirements, modDepthFromGoVersion(ld.GoVersion))
   945  	if err != nil {
   946  		ld.errorf("go: %v\n", err)
   947  	}
   948  
   949  	if ld.requirements.depth == eager {
   950  		var err error
   951  		ld.requirements, _, err = expandGraph(ctx, ld.requirements)
   952  		if err != nil {
   953  			ld.errorf("go: %v\n", err)
   954  		}
   955  	}
   956  
   957  	for {
   958  		ld.reset()
   959  
   960  		// Load the root packages and their imports.
   961  		// Note: the returned roots can change on each iteration,
   962  		// since the expansion of package patterns depends on the
   963  		// build list we're using.
   964  		rootPkgs := ld.listRoots(ld.requirements)
   965  
   966  		if ld.requirements.depth == lazy && cfg.BuildMod == "mod" {
   967  			// Before we start loading transitive imports of packages, locate all of
   968  			// the root packages and promote their containing modules to root modules
   969  			// dependencies. If their go.mod files are tidy (the common case) and the
   970  			// set of root packages does not change then we can select the correct
   971  			// versions of all transitive imports on the first try and complete
   972  			// loading in a single iteration.
   973  			changedBuildList := ld.preloadRootModules(ctx, rootPkgs)
   974  			if changedBuildList {
   975  				// The build list has changed, so the set of root packages may have also
   976  				// changed. Start over to pick up the changes. (Preloading roots is much
   977  				// cheaper than loading the full import graph, so we would rather pay
   978  				// for an extra iteration of preloading than potentially end up
   979  				// discarding the result of a full iteration of loading.)
   980  				continue
   981  			}
   982  		}
   983  
   984  		inRoots := map[*loadPkg]bool{}
   985  		for _, path := range rootPkgs {
   986  			root := ld.pkg(ctx, path, pkgIsRoot)
   987  			if !inRoots[root] {
   988  				ld.roots = append(ld.roots, root)
   989  				inRoots[root] = true
   990  			}
   991  		}
   992  
   993  		// ld.pkg adds imported packages to the work queue and calls applyPkgFlags,
   994  		// which adds tests (and test dependencies) as needed.
   995  		//
   996  		// When all of the work in the queue has completed, we'll know that the
   997  		// transitive closure of dependencies has been loaded.
   998  		<-ld.work.Idle()
   999  
  1000  		ld.buildStacks()
  1001  
  1002  		changed, err := ld.updateRequirements(ctx)
  1003  		if err != nil {
  1004  			ld.errorf("go: %v\n", err)
  1005  			break
  1006  		}
  1007  		if changed {
  1008  			// Don't resolve missing imports until the module graph have stabilized.
  1009  			// If the roots are still changing, they may turn out to specify a
  1010  			// requirement on the missing package(s), and we would rather use a
  1011  			// version specified by a new root than add a new dependency on an
  1012  			// unrelated version.
  1013  			continue
  1014  		}
  1015  
  1016  		if !ld.ResolveMissingImports || (!HasModRoot() && !allowMissingModuleImports) {
  1017  			// We've loaded as much as we can without resolving missing imports.
  1018  			break
  1019  		}
  1020  
  1021  		modAddedBy := ld.resolveMissingImports(ctx)
  1022  		if len(modAddedBy) == 0 {
  1023  			// The roots are stable, and we've resolved all of the missing packages
  1024  			// that we can.
  1025  			break
  1026  		}
  1027  
  1028  		toAdd := make([]module.Version, 0, len(modAddedBy))
  1029  		for m, _ := range modAddedBy {
  1030  			toAdd = append(toAdd, m)
  1031  		}
  1032  		module.Sort(toAdd) // to make errors deterministic
  1033  
  1034  		// We ran updateRequirements before resolving missing imports and it didn't
  1035  		// make any changes, so we know that the requirement graph is already
  1036  		// consistent with ld.pkgs: we don't need to pass ld.pkgs to updateRoots
  1037  		// again. (That would waste time looking for changes that we have already
  1038  		// applied.)
  1039  		var noPkgs []*loadPkg
  1040  		// We also know that we're going to call updateRequirements again next
  1041  		// iteration so we don't need to also update it here. (That would waste time
  1042  		// computing a "direct" map that we'll have to recompute later anyway.)
  1043  		direct := ld.requirements.direct
  1044  		rs, err := updateRoots(ctx, direct, ld.requirements, noPkgs, toAdd, ld.AssumeRootsImported)
  1045  		if err != nil {
  1046  			// If an error was found in a newly added module, report the package
  1047  			// import stack instead of the module requirement stack. Packages
  1048  			// are more descriptive.
  1049  			if err, ok := err.(*mvs.BuildListError); ok {
  1050  				if pkg := modAddedBy[err.Module()]; pkg != nil {
  1051  					ld.errorf("go: %s: %v\n", pkg.stackText(), err.Err)
  1052  					break
  1053  				}
  1054  			}
  1055  			ld.errorf("go: %v\n", err)
  1056  			break
  1057  		}
  1058  		if reflect.DeepEqual(rs.rootModules, ld.requirements.rootModules) {
  1059  			// Something is deeply wrong. resolveMissingImports gave us a non-empty
  1060  			// set of modules to add to the graph, but adding those modules had no
  1061  			// effect — either they were already in the graph, or updateRoots did not
  1062  			// add them as requested.
  1063  			panic(fmt.Sprintf("internal error: adding %v to module graph had no effect on root requirements (%v)", toAdd, rs.rootModules))
  1064  		}
  1065  		ld.requirements = rs
  1066  	}
  1067  	base.ExitIfErrors() // TODO(bcmills): Is this actually needed?
  1068  
  1069  	// Tidy the build list, if applicable, before we report errors.
  1070  	// (The process of tidying may remove errors from irrelevant dependencies.)
  1071  	if ld.Tidy {
  1072  		rs, err := tidyRoots(ctx, ld.requirements, ld.pkgs)
  1073  		if err != nil {
  1074  			ld.errorf("go: %v\n", err)
  1075  		}
  1076  
  1077  		if ld.requirements.depth == lazy {
  1078  			// We continuously add tidy roots to ld.requirements during loading, so at
  1079  			// this point the tidy roots should be a subset of the roots of
  1080  			// ld.requirements, ensuring that no new dependencies are brought inside
  1081  			// the lazy-loading horizon.
  1082  			// If that is not the case, there is a bug in the loading loop above.
  1083  			for _, m := range rs.rootModules {
  1084  				if v, ok := ld.requirements.rootSelected(m.Path); !ok || v != m.Version {
  1085  					ld.errorf("go mod tidy: internal error: a requirement on %v is needed but was not added during package loading\n", m)
  1086  					base.ExitIfErrors()
  1087  				}
  1088  			}
  1089  		}
  1090  		ld.requirements = rs
  1091  	}
  1092  
  1093  	// Report errors, if any.
  1094  	for _, pkg := range ld.pkgs {
  1095  		if pkg.err == nil {
  1096  			continue
  1097  		}
  1098  
  1099  		// Add importer information to checksum errors.
  1100  		if sumErr := (*ImportMissingSumError)(nil); errors.As(pkg.err, &sumErr) {
  1101  			if importer := pkg.stack; importer != nil {
  1102  				sumErr.importer = importer.path
  1103  				sumErr.importerVersion = importer.mod.Version
  1104  				sumErr.importerIsTest = importer.testOf != nil
  1105  			}
  1106  		}
  1107  
  1108  		if ld.SilencePackageErrors {
  1109  			continue
  1110  		}
  1111  		if stdErr := (*ImportMissingError)(nil); errors.As(pkg.err, &stdErr) &&
  1112  			stdErr.isStd && ld.SilenceMissingStdImports {
  1113  			continue
  1114  		}
  1115  		if ld.SilenceNoGoErrors && errors.Is(pkg.err, imports.ErrNoGo) {
  1116  			continue
  1117  		}
  1118  
  1119  		ld.errorf("%s: %v\n", pkg.stackText(), pkg.err)
  1120  	}
  1121  
  1122  	ld.checkMultiplePaths()
  1123  	return ld
  1124  }
  1125  
  1126  // updateRequirements ensures that ld.requirements is consistent with the
  1127  // information gained from ld.pkgs and includes the modules in add as roots at
  1128  // at least the given versions.
  1129  //
  1130  // In particular:
  1131  //
  1132  // 	- Modules that provide packages directly imported from the main module are
  1133  // 	  marked as direct, and are promoted to explicit roots. If a needed root
  1134  // 	  cannot be promoted due to -mod=readonly or -mod=vendor, the importing
  1135  // 	  package is marked with an error.
  1136  //
  1137  // 	- If ld scanned the "all" pattern independent of build constraints, it is
  1138  // 	  guaranteed to have seen every direct import. Module dependencies that did
  1139  // 	  not provide any directly-imported package are then marked as indirect.
  1140  //
  1141  // 	- Root dependencies are updated to their selected versions.
  1142  //
  1143  // The "changed" return value reports whether the update changed the selected
  1144  // version of any module that either provided a loaded package or may now
  1145  // provide a package that was previously unresolved.
  1146  func (ld *loader) updateRequirements(ctx context.Context) (changed bool, err error) {
  1147  	rs := ld.requirements
  1148  
  1149  	// direct contains the set of modules believed to provide packages directly
  1150  	// imported by the main module.
  1151  	var direct map[string]bool
  1152  
  1153  	// If we didn't scan all of the imports from the main module, or didn't use
  1154  	// imports.AnyTags, then we didn't necessarily load every package that
  1155  	// contributes “direct” imports — so we can't safely mark existing direct
  1156  	// dependencies in ld.requirements as indirect-only. Propagate them as direct.
  1157  	loadedDirect := ld.allPatternIsRoot && reflect.DeepEqual(ld.Tags, imports.AnyTags())
  1158  	if loadedDirect {
  1159  		direct = make(map[string]bool)
  1160  	} else {
  1161  		// TODO(bcmills): It seems like a shame to allocate and copy a map here when
  1162  		// it will only rarely actually vary from rs.direct. Measure this cost and
  1163  		// maybe avoid the copy.
  1164  		direct = make(map[string]bool, len(rs.direct))
  1165  		for mPath := range rs.direct {
  1166  			direct[mPath] = true
  1167  		}
  1168  	}
  1169  
  1170  	for _, pkg := range ld.pkgs {
  1171  		if pkg.mod != Target {
  1172  			continue
  1173  		}
  1174  		for _, dep := range pkg.imports {
  1175  			if !dep.fromExternalModule() {
  1176  				continue
  1177  			}
  1178  
  1179  			if pkg.err == nil && cfg.BuildMod != "mod" {
  1180  				if v, ok := rs.rootSelected(dep.mod.Path); !ok || v != dep.mod.Version {
  1181  					// dep.mod is not an explicit dependency, but needs to be.
  1182  					// Because we are not in "mod" mode, we will not be able to update it.
  1183  					// Instead, mark the importing package with an error.
  1184  					//
  1185  					// TODO(#41688): The resulting error message fails to include the file
  1186  					// position of the import statement (because that information is not
  1187  					// tracked by the module loader). Figure out how to plumb the import
  1188  					// position through.
  1189  					pkg.err = &DirectImportFromImplicitDependencyError{
  1190  						ImporterPath: pkg.path,
  1191  						ImportedPath: dep.path,
  1192  						Module:       dep.mod,
  1193  					}
  1194  					// cfg.BuildMod does not allow us to change dep.mod to be a direct
  1195  					// dependency, so don't mark it as such.
  1196  					continue
  1197  				}
  1198  			}
  1199  
  1200  			// dep is a package directly imported by a package or test in the main
  1201  			// module and loaded from some other module (not the standard library).
  1202  			// Mark its module as a direct dependency.
  1203  			direct[dep.mod.Path] = true
  1204  		}
  1205  	}
  1206  
  1207  	var addRoots []module.Version
  1208  	if ld.Tidy {
  1209  		// When we are tidying a lazy module, we may need to add roots to preserve
  1210  		// the versions of indirect, test-only dependencies that are upgraded
  1211  		// above or otherwise missing from the go.mod files of direct
  1212  		// dependencies. (For example, the direct dependency might be a very
  1213  		// stable codebase that predates modules and thus lacks a go.mod file, or
  1214  		// the author of the direct dependency may have forgotten to commit a
  1215  		// change to the go.mod file, or may have made an erroneous hand-edit that
  1216  		// causes it to be untidy.)
  1217  		//
  1218  		// Promoting an indirect dependency to a root adds the next layer of its
  1219  		// dependencies to the module graph, which may increase the selected
  1220  		// versions of other modules from which we have already loaded packages.
  1221  		// So after we promote an indirect dependency to a root, we need to reload
  1222  		// packages, which means another iteration of loading.
  1223  		//
  1224  		// As an extra wrinkle, the upgrades due to promoting a root can cause
  1225  		// previously-resolved packages to become unresolved. For example, the
  1226  		// module providing an unstable package might be upgraded to a version
  1227  		// that no longer contains that package. If we then resolve the missing
  1228  		// package, we might add yet another root that upgrades away some other
  1229  		// dependency. (The tests in mod_tidy_convergence*.txt illustrate some
  1230  		// particularly worrisome cases.)
  1231  		//
  1232  		// To ensure that this process of promoting, adding, and upgrading roots
  1233  		// eventually terminates, during iteration we only ever add modules to the
  1234  		// root set — we only remove irrelevant roots at the very end of
  1235  		// iteration, after we have already added every root that we plan to need
  1236  		// in the (eventual) tidy root set.
  1237  		//
  1238  		// Since we do not remove any roots during iteration, even if they no
  1239  		// longer provide any imported packages, the selected versions of the
  1240  		// roots can only increase and the set of roots can only expand. The set
  1241  		// of extant root paths is finite and the set of versions of each path is
  1242  		// finite, so the iteration *must* reach a stable fixed-point.
  1243  		tidy, err := tidyRoots(ctx, rs, ld.pkgs)
  1244  		if err != nil {
  1245  			return false, err
  1246  		}
  1247  		addRoots = tidy.rootModules
  1248  	}
  1249  
  1250  	rs, err = updateRoots(ctx, direct, rs, ld.pkgs, addRoots, ld.AssumeRootsImported)
  1251  	if err != nil {
  1252  		// We don't actually know what even the root requirements are supposed to be,
  1253  		// so we can't proceed with loading. Return the error to the caller
  1254  		return false, err
  1255  	}
  1256  
  1257  	if rs != ld.requirements && !reflect.DeepEqual(rs.rootModules, ld.requirements.rootModules) {
  1258  		// The roots of the module graph have changed in some way (not just the
  1259  		// "direct" markings). Check whether the changes affected any of the loaded
  1260  		// packages.
  1261  		mg, err := rs.Graph(ctx)
  1262  		if err != nil {
  1263  			return false, err
  1264  		}
  1265  		for _, pkg := range ld.pkgs {
  1266  			if pkg.fromExternalModule() && mg.Selected(pkg.mod.Path) != pkg.mod.Version {
  1267  				changed = true
  1268  				break
  1269  			}
  1270  			if pkg.err != nil {
  1271  				// Promoting a module to a root may resolve an import that was
  1272  				// previously missing (by pulling in a previously-prune dependency that
  1273  				// provides it) or ambiguous (by promoting exactly one of the
  1274  				// alternatives to a root and ignoring the second-level alternatives) or
  1275  				// otherwise errored out (by upgrading from a version that cannot be
  1276  				// fetched to one that can be).
  1277  				//
  1278  				// Instead of enumerating all of the possible errors, we'll just check
  1279  				// whether importFromModules returns nil for the package.
  1280  				// False-positives are ok: if we have a false-positive here, we'll do an
  1281  				// extra iteration of package loading this time, but we'll still
  1282  				// converge when the root set stops changing.
  1283  				//
  1284  				// In some sense, we can think of this as ‘upgraded the module providing
  1285  				// pkg.path from "none" to a version higher than "none"’.
  1286  				if _, _, err = importFromModules(ctx, pkg.path, rs, nil); err == nil {
  1287  					changed = true
  1288  					break
  1289  				}
  1290  			}
  1291  		}
  1292  	}
  1293  
  1294  	ld.requirements = rs
  1295  	return changed, nil
  1296  }
  1297  
  1298  // resolveMissingImports returns a set of modules that could be added as
  1299  // dependencies in order to resolve missing packages from pkgs.
  1300  //
  1301  // The newly-resolved packages are added to the addedModuleFor map, and
  1302  // resolveMissingImports returns a map from each new module version to
  1303  // the first missing package that module would resolve.
  1304  func (ld *loader) resolveMissingImports(ctx context.Context) (modAddedBy map[module.Version]*loadPkg) {
  1305  	type pkgMod struct {
  1306  		pkg *loadPkg
  1307  		mod *module.Version
  1308  	}
  1309  	var pkgMods []pkgMod
  1310  	for _, pkg := range ld.pkgs {
  1311  		if pkg.err == nil {
  1312  			continue
  1313  		}
  1314  		if pkg.isTest() {
  1315  			// If we are missing a test, we are also missing its non-test version, and
  1316  			// we should only add the missing import once.
  1317  			continue
  1318  		}
  1319  		if !errors.As(pkg.err, new(*ImportMissingError)) {
  1320  			// Leave other errors for Import or load.Packages to report.
  1321  			continue
  1322  		}
  1323  
  1324  		pkg := pkg
  1325  		var mod module.Version
  1326  		ld.work.Add(func() {
  1327  			var err error
  1328  			mod, err = queryImport(ctx, pkg.path, ld.requirements)
  1329  			if err != nil {
  1330  				// pkg.err was already non-nil, so we can reasonably attribute the error
  1331  				// for pkg to either the original error or the one returned by
  1332  				// queryImport. The existing error indicates only that we couldn't find
  1333  				// the package, whereas the query error also explains why we didn't fix
  1334  				// the problem — so we prefer the latter.
  1335  				pkg.err = err
  1336  			}
  1337  
  1338  			// err is nil, but we intentionally leave pkg.err non-nil and pkg.mod
  1339  			// unset: we still haven't satisfied other invariants of a
  1340  			// successfully-loaded package, such as scanning and loading the imports
  1341  			// of that package. If we succeed in resolving the new dependency graph,
  1342  			// the caller can reload pkg and update the error at that point.
  1343  			//
  1344  			// Even then, the package might not be loaded from the version we've
  1345  			// identified here. The module may be upgraded by some other dependency,
  1346  			// or by a transitive dependency of mod itself, or — less likely — the
  1347  			// package may be rejected by an AllowPackage hook or rendered ambiguous
  1348  			// by some other newly-added or newly-upgraded dependency.
  1349  		})
  1350  
  1351  		pkgMods = append(pkgMods, pkgMod{pkg: pkg, mod: &mod})
  1352  	}
  1353  	<-ld.work.Idle()
  1354  
  1355  	modAddedBy = map[module.Version]*loadPkg{}
  1356  	for _, pm := range pkgMods {
  1357  		pkg, mod := pm.pkg, *pm.mod
  1358  		if mod.Path == "" {
  1359  			continue
  1360  		}
  1361  
  1362  		fmt.Fprintf(os.Stderr, "go: found %s in %s %s\n", pkg.path, mod.Path, mod.Version)
  1363  		if modAddedBy[mod] == nil {
  1364  			modAddedBy[mod] = pkg
  1365  		}
  1366  	}
  1367  
  1368  	return modAddedBy
  1369  }
  1370  
  1371  // pkg locates the *loadPkg for path, creating and queuing it for loading if
  1372  // needed, and updates its state to reflect the given flags.
  1373  //
  1374  // The imports of the returned *loadPkg will be loaded asynchronously in the
  1375  // ld.work queue, and its test (if requested) will also be populated once
  1376  // imports have been resolved. When ld.work goes idle, all transitive imports of
  1377  // the requested package (and its test, if requested) will have been loaded.
  1378  func (ld *loader) pkg(ctx context.Context, path string, flags loadPkgFlags) *loadPkg {
  1379  	if flags.has(pkgImportsLoaded) {
  1380  		panic("internal error: (*loader).pkg called with pkgImportsLoaded flag set")
  1381  	}
  1382  
  1383  	pkg := ld.pkgCache.Do(path, func() interface{} {
  1384  		pkg := &loadPkg{
  1385  			path: path,
  1386  		}
  1387  		ld.applyPkgFlags(ctx, pkg, flags)
  1388  
  1389  		ld.work.Add(func() { ld.load(ctx, pkg) })
  1390  		return pkg
  1391  	}).(*loadPkg)
  1392  
  1393  	ld.applyPkgFlags(ctx, pkg, flags)
  1394  	return pkg
  1395  }
  1396  
  1397  // applyPkgFlags updates pkg.flags to set the given flags and propagate the
  1398  // (transitive) effects of those flags, possibly loading or enqueueing further
  1399  // packages as a result.
  1400  func (ld *loader) applyPkgFlags(ctx context.Context, pkg *loadPkg, flags loadPkgFlags) {
  1401  	if flags == 0 {
  1402  		return
  1403  	}
  1404  
  1405  	if flags.has(pkgInAll) && ld.allPatternIsRoot && !pkg.isTest() {
  1406  		// This package matches a root pattern by virtue of being in "all".
  1407  		flags |= pkgIsRoot
  1408  	}
  1409  	if flags.has(pkgIsRoot) {
  1410  		flags |= pkgFromRoot
  1411  	}
  1412  
  1413  	old := pkg.flags.update(flags)
  1414  	new := old | flags
  1415  	if new == old || !new.has(pkgImportsLoaded) {
  1416  		// We either didn't change the state of pkg, or we don't know anything about
  1417  		// its dependencies yet. Either way, we can't usefully load its test or
  1418  		// update its dependencies.
  1419  		return
  1420  	}
  1421  
  1422  	if !pkg.isTest() {
  1423  		// Check whether we should add (or update the flags for) a test for pkg.
  1424  		// ld.pkgTest is idempotent and extra invocations are inexpensive,
  1425  		// so it's ok if we call it more than is strictly necessary.
  1426  		wantTest := false
  1427  		switch {
  1428  		case ld.allPatternIsRoot && pkg.mod == Target:
  1429  			// We are loading the "all" pattern, which includes packages imported by
  1430  			// tests in the main module. This package is in the main module, so we
  1431  			// need to identify the imports of its test even if LoadTests is not set.
  1432  			//
  1433  			// (We will filter out the extra tests explicitly in computePatternAll.)
  1434  			wantTest = true
  1435  
  1436  		case ld.allPatternIsRoot && ld.allClosesOverTests && new.has(pkgInAll):
  1437  			// This variant of the "all" pattern includes imports of tests of every
  1438  			// package that is itself in "all", and pkg is in "all", so its test is
  1439  			// also in "all" (as above).
  1440  			wantTest = true
  1441  
  1442  		case ld.LoadTests && new.has(pkgIsRoot):
  1443  			// LoadTest explicitly requests tests of “the root packages”.
  1444  			wantTest = true
  1445  		}
  1446  
  1447  		if wantTest {
  1448  			var testFlags loadPkgFlags
  1449  			if pkg.mod == Target || (ld.allClosesOverTests && new.has(pkgInAll)) {
  1450  				// Tests of packages in the main module are in "all", in the sense that
  1451  				// they cause the packages they import to also be in "all". So are tests
  1452  				// of packages in "all" if "all" closes over test dependencies.
  1453  				testFlags |= pkgInAll
  1454  			}
  1455  			ld.pkgTest(ctx, pkg, testFlags)
  1456  		}
  1457  	}
  1458  
  1459  	if new.has(pkgInAll) && !old.has(pkgInAll|pkgImportsLoaded) {
  1460  		// We have just marked pkg with pkgInAll, or we have just loaded its
  1461  		// imports, or both. Now is the time to propagate pkgInAll to the imports.
  1462  		for _, dep := range pkg.imports {
  1463  			ld.applyPkgFlags(ctx, dep, pkgInAll)
  1464  		}
  1465  	}
  1466  
  1467  	if new.has(pkgFromRoot) && !old.has(pkgFromRoot|pkgImportsLoaded) {
  1468  		for _, dep := range pkg.imports {
  1469  			ld.applyPkgFlags(ctx, dep, pkgFromRoot)
  1470  		}
  1471  	}
  1472  }
  1473  
  1474  // preloadRootModules loads the module requirements needed to identify the
  1475  // selected version of each module providing a package in rootPkgs,
  1476  // adding new root modules to the module graph if needed.
  1477  func (ld *loader) preloadRootModules(ctx context.Context, rootPkgs []string) (changedBuildList bool) {
  1478  	needc := make(chan map[module.Version]bool, 1)
  1479  	needc <- map[module.Version]bool{}
  1480  	for _, path := range rootPkgs {
  1481  		path := path
  1482  		ld.work.Add(func() {
  1483  			// First, try to identify the module containing the package using only roots.
  1484  			//
  1485  			// If the main module is tidy and the package is in "all" — or if we're
  1486  			// lucky — we can identify all of its imports without actually loading the
  1487  			// full module graph.
  1488  			m, _, err := importFromModules(ctx, path, ld.requirements, nil)
  1489  			if err != nil {
  1490  				var missing *ImportMissingError
  1491  				if errors.As(err, &missing) && ld.ResolveMissingImports {
  1492  					// This package isn't provided by any selected module.
  1493  					// If we can find it, it will be a new root dependency.
  1494  					m, err = queryImport(ctx, path, ld.requirements)
  1495  				}
  1496  				if err != nil {
  1497  					// We couldn't identify the root module containing this package.
  1498  					// Leave it unresolved; we will report it during loading.
  1499  					return
  1500  				}
  1501  			}
  1502  			if m.Path == "" {
  1503  				// The package is in std or cmd. We don't need to change the root set.
  1504  				return
  1505  			}
  1506  
  1507  			v, ok := ld.requirements.rootSelected(m.Path)
  1508  			if !ok || v != m.Version {
  1509  				// We found the requested package in m, but m is not a root, so
  1510  				// loadModGraph will not load its requirements. We need to promote the
  1511  				// module to a root to ensure that any other packages this package
  1512  				// imports are resolved from correct dependency versions.
  1513  				//
  1514  				// (This is the “argument invariant” from the lazy loading design.)
  1515  				need := <-needc
  1516  				need[m] = true
  1517  				needc <- need
  1518  			}
  1519  		})
  1520  	}
  1521  	<-ld.work.Idle()
  1522  
  1523  	need := <-needc
  1524  	if len(need) == 0 {
  1525  		return false // No roots to add.
  1526  	}
  1527  
  1528  	toAdd := make([]module.Version, 0, len(need))
  1529  	for m := range need {
  1530  		toAdd = append(toAdd, m)
  1531  	}
  1532  	module.Sort(toAdd)
  1533  
  1534  	rs, err := updateRoots(ctx, ld.requirements.direct, ld.requirements, nil, toAdd, ld.AssumeRootsImported)
  1535  	if err != nil {
  1536  		// We are missing some root dependency, and for some reason we can't load
  1537  		// enough of the module dependency graph to add the missing root. Package
  1538  		// loading is doomed to fail, so fail quickly.
  1539  		ld.errorf("go: %v\n", err)
  1540  		base.ExitIfErrors()
  1541  		return false
  1542  	}
  1543  	if reflect.DeepEqual(rs.rootModules, ld.requirements.rootModules) {
  1544  		// Something is deeply wrong. resolveMissingImports gave us a non-empty
  1545  		// set of modules to add to the graph, but adding those modules had no
  1546  		// effect — either they were already in the graph, or updateRoots did not
  1547  		// add them as requested.
  1548  		panic(fmt.Sprintf("internal error: adding %v to module graph had no effect on root requirements (%v)", toAdd, rs.rootModules))
  1549  	}
  1550  
  1551  	ld.requirements = rs
  1552  	return true
  1553  }
  1554  
  1555  // load loads an individual package.
  1556  func (ld *loader) load(ctx context.Context, pkg *loadPkg) {
  1557  	if strings.Contains(pkg.path, "@") {
  1558  		// Leave for error during load.
  1559  		return
  1560  	}
  1561  	if build.IsLocalImport(pkg.path) || filepath.IsAbs(pkg.path) {
  1562  		// Leave for error during load.
  1563  		// (Module mode does not allow local imports.)
  1564  		return
  1565  	}
  1566  
  1567  	if search.IsMetaPackage(pkg.path) {
  1568  		pkg.err = &invalidImportError{
  1569  			importPath: pkg.path,
  1570  			err:        fmt.Errorf("%q is not an importable package; see 'go help packages'", pkg.path),
  1571  		}
  1572  		return
  1573  	}
  1574  
  1575  	var mg *ModuleGraph
  1576  	if ld.requirements.depth == eager {
  1577  		var err error
  1578  		mg, err = ld.requirements.Graph(ctx)
  1579  		if err != nil {
  1580  			// We already checked the error from Graph in loadFromRoots and/or
  1581  			// updateRequirements, so we ignored the error on purpose and we should
  1582  			// keep trying to push past it.
  1583  			//
  1584  			// However, because mg may be incomplete (and thus may select inaccurate
  1585  			// versions), we shouldn't use it to load packages. Instead, we pass a nil
  1586  			// *ModuleGraph, which will cause mg to first try loading from only the
  1587  			// main module and root dependencies.
  1588  			mg = nil
  1589  		}
  1590  	}
  1591  
  1592  	pkg.mod, pkg.dir, pkg.err = importFromModules(ctx, pkg.path, ld.requirements, mg)
  1593  	if pkg.dir == "" {
  1594  		return
  1595  	}
  1596  	if pkg.mod == Target {
  1597  		// Go ahead and mark pkg as in "all". This provides the invariant that a
  1598  		// package that is *only* imported by other packages in "all" is always
  1599  		// marked as such before loading its imports.
  1600  		//
  1601  		// We don't actually rely on that invariant at the moment, but it may
  1602  		// improve efficiency somewhat and makes the behavior a bit easier to reason
  1603  		// about (by reducing churn on the flag bits of dependencies), and costs
  1604  		// essentially nothing (these atomic flag ops are essentially free compared
  1605  		// to scanning source code for imports).
  1606  		ld.applyPkgFlags(ctx, pkg, pkgInAll)
  1607  	}
  1608  	if ld.AllowPackage != nil {
  1609  		if err := ld.AllowPackage(ctx, pkg.path, pkg.mod); err != nil {
  1610  			pkg.err = err
  1611  		}
  1612  	}
  1613  
  1614  	pkg.inStd = (search.IsStandardImportPath(pkg.path) && search.InDir(pkg.dir, cfg.GOROOTsrc) != "")
  1615  
  1616  	var imports, testImports []string
  1617  
  1618  	if cfg.BuildContext.Compiler == "gccgo" && pkg.inStd {
  1619  		// We can't scan standard packages for gccgo.
  1620  	} else {
  1621  		var err error
  1622  		imports, testImports, err = scanDir(pkg.dir, ld.Tags)
  1623  		if err != nil {
  1624  			pkg.err = err
  1625  			return
  1626  		}
  1627  	}
  1628  
  1629  	pkg.imports = make([]*loadPkg, 0, len(imports))
  1630  	var importFlags loadPkgFlags
  1631  	if pkg.flags.has(pkgInAll) {
  1632  		importFlags = pkgInAll
  1633  	}
  1634  	for _, path := range imports {
  1635  		if pkg.inStd {
  1636  			// Imports from packages in "std" and "cmd" should resolve using
  1637  			// GOROOT/src/vendor even when "std" is not the main module.
  1638  			path = ld.stdVendor(pkg.path, path)
  1639  		}
  1640  		pkg.imports = append(pkg.imports, ld.pkg(ctx, path, importFlags))
  1641  	}
  1642  	pkg.testImports = testImports
  1643  
  1644  	ld.applyPkgFlags(ctx, pkg, pkgImportsLoaded)
  1645  }
  1646  
  1647  // pkgTest locates the test of pkg, creating it if needed, and updates its state
  1648  // to reflect the given flags.
  1649  //
  1650  // pkgTest requires that the imports of pkg have already been loaded (flagged
  1651  // with pkgImportsLoaded).
  1652  func (ld *loader) pkgTest(ctx context.Context, pkg *loadPkg, testFlags loadPkgFlags) *loadPkg {
  1653  	if pkg.isTest() {
  1654  		panic("pkgTest called on a test package")
  1655  	}
  1656  
  1657  	createdTest := false
  1658  	pkg.testOnce.Do(func() {
  1659  		pkg.test = &loadPkg{
  1660  			path:   pkg.path,
  1661  			testOf: pkg,
  1662  			mod:    pkg.mod,
  1663  			dir:    pkg.dir,
  1664  			err:    pkg.err,
  1665  			inStd:  pkg.inStd,
  1666  		}
  1667  		ld.applyPkgFlags(ctx, pkg.test, testFlags)
  1668  		createdTest = true
  1669  	})
  1670  
  1671  	test := pkg.test
  1672  	if createdTest {
  1673  		test.imports = make([]*loadPkg, 0, len(pkg.testImports))
  1674  		var importFlags loadPkgFlags
  1675  		if test.flags.has(pkgInAll) {
  1676  			importFlags = pkgInAll
  1677  		}
  1678  		for _, path := range pkg.testImports {
  1679  			if pkg.inStd {
  1680  				path = ld.stdVendor(test.path, path)
  1681  			}
  1682  			test.imports = append(test.imports, ld.pkg(ctx, path, importFlags))
  1683  		}
  1684  		pkg.testImports = nil
  1685  		ld.applyPkgFlags(ctx, test, pkgImportsLoaded)
  1686  	} else {
  1687  		ld.applyPkgFlags(ctx, test, testFlags)
  1688  	}
  1689  
  1690  	return test
  1691  }
  1692  
  1693  // stdVendor returns the canonical import path for the package with the given
  1694  // path when imported from the standard-library package at parentPath.
  1695  func (ld *loader) stdVendor(parentPath, path string) string {
  1696  	if search.IsStandardImportPath(path) {
  1697  		return path
  1698  	}
  1699  
  1700  	if str.HasPathPrefix(parentPath, "cmd") {
  1701  		if !ld.VendorModulesInGOROOTSrc || Target.Path != "cmd" {
  1702  			vendorPath := pathpkg.Join("cmd", "vendor", path)
  1703  			if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil {
  1704  				return vendorPath
  1705  			}
  1706  		}
  1707  	} else if !ld.VendorModulesInGOROOTSrc || Target.Path != "std" || str.HasPathPrefix(parentPath, "vendor") {
  1708  		// If we are outside of the 'std' module, resolve imports from within 'std'
  1709  		// to the vendor directory.
  1710  		//
  1711  		// Do the same for importers beginning with the prefix 'vendor/' even if we
  1712  		// are *inside* of the 'std' module: the 'vendor/' packages that resolve
  1713  		// globally from GOROOT/src/vendor (and are listed as part of 'go list std')
  1714  		// are distinct from the real module dependencies, and cannot import
  1715  		// internal packages from the real module.
  1716  		//
  1717  		// (Note that although the 'vendor/' packages match the 'std' *package*
  1718  		// pattern, they are not part of the std *module*, and do not affect
  1719  		// 'go mod tidy' and similar module commands when working within std.)
  1720  		vendorPath := pathpkg.Join("vendor", path)
  1721  		if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil {
  1722  			return vendorPath
  1723  		}
  1724  	}
  1725  
  1726  	// Not vendored: resolve from modules.
  1727  	return path
  1728  }
  1729  
  1730  // computePatternAll returns the list of packages matching pattern "all",
  1731  // starting with a list of the import paths for the packages in the main module.
  1732  func (ld *loader) computePatternAll() (all []string) {
  1733  	for _, pkg := range ld.pkgs {
  1734  		if pkg.flags.has(pkgInAll) && !pkg.isTest() {
  1735  			all = append(all, pkg.path)
  1736  		}
  1737  	}
  1738  	sort.Strings(all)
  1739  	return all
  1740  }
  1741  
  1742  // checkMultiplePaths verifies that a given module path is used as itself
  1743  // or as a replacement for another module, but not both at the same time.
  1744  //
  1745  // (See https://golang.org/issue/26607 and https://golang.org/issue/34650.)
  1746  func (ld *loader) checkMultiplePaths() {
  1747  	mods := ld.requirements.rootModules
  1748  	if cached := ld.requirements.graph.Load(); cached != nil {
  1749  		if mg := cached.(cachedGraph).mg; mg != nil {
  1750  			mods = mg.BuildList()
  1751  		}
  1752  	}
  1753  
  1754  	firstPath := map[module.Version]string{}
  1755  	for _, mod := range mods {
  1756  		src := resolveReplacement(mod)
  1757  		if prev, ok := firstPath[src]; !ok {
  1758  			firstPath[src] = mod.Path
  1759  		} else if prev != mod.Path {
  1760  			ld.errorf("go: %s@%s used for two different module paths (%s and %s)\n", src.Path, src.Version, prev, mod.Path)
  1761  		}
  1762  	}
  1763  }
  1764  
  1765  // checkTidyCompatibility emits an error if any package would be loaded from a
  1766  // different module under rs than under ld.requirements.
  1767  func (ld *loader) checkTidyCompatibility(ctx context.Context, rs *Requirements) {
  1768  	suggestUpgrade := false
  1769  	suggestEFlag := false
  1770  	suggestFixes := func() {
  1771  		if ld.AllowErrors {
  1772  			// The user is explicitly ignoring these errors, so don't bother them with
  1773  			// other options.
  1774  			return
  1775  		}
  1776  
  1777  		// We print directly to os.Stderr because this information is advice about
  1778  		// how to fix errors, not actually an error itself.
  1779  		// (The actual errors should have been logged already.)
  1780  
  1781  		fmt.Fprintln(os.Stderr)
  1782  
  1783  		goFlag := ""
  1784  		if ld.GoVersion != modFileGoVersion() {
  1785  			goFlag = " -go=" + ld.GoVersion
  1786  		}
  1787  
  1788  		compatFlag := ""
  1789  		if ld.TidyCompatibleVersion != priorGoVersion(ld.GoVersion) {
  1790  			compatFlag = " -compat=" + ld.TidyCompatibleVersion
  1791  		}
  1792  		if suggestUpgrade {
  1793  			eDesc := ""
  1794  			eFlag := ""
  1795  			if suggestEFlag {
  1796  				eDesc = ", leaving some packages unresolved"
  1797  				eFlag = " -e"
  1798  			}
  1799  			fmt.Fprintf(os.Stderr, "To upgrade to the versions selected by go %s%s:\n\tgo mod tidy%s -go=%s && go mod tidy%s -go=%s%s\n", ld.TidyCompatibleVersion, eDesc, eFlag, ld.TidyCompatibleVersion, eFlag, ld.GoVersion, compatFlag)
  1800  		} else if suggestEFlag {
  1801  			// If some packages are missing but no package is upgraded, then we
  1802  			// shouldn't suggest upgrading to the Go 1.16 versions explicitly — that
  1803  			// wouldn't actually fix anything for Go 1.16 users, and *would* break
  1804  			// something for Go 1.17 users.
  1805  			fmt.Fprintf(os.Stderr, "To proceed despite packages unresolved in go %s:\n\tgo mod tidy -e%s%s\n", ld.TidyCompatibleVersion, goFlag, compatFlag)
  1806  		}
  1807  
  1808  		fmt.Fprintf(os.Stderr, "If reproducibility with go %s is not needed:\n\tgo mod tidy%s -compat=%s\n", ld.TidyCompatibleVersion, goFlag, ld.GoVersion)
  1809  
  1810  		// TODO(#46141): Populate the linked wiki page.
  1811  		fmt.Fprintf(os.Stderr, "For other options, see:\n\thttps://golang.org/doc/modules/pruning\n")
  1812  	}
  1813  
  1814  	mg, err := rs.Graph(ctx)
  1815  	if err != nil {
  1816  		ld.errorf("go mod tidy: error loading go %s module graph: %v\n", ld.TidyCompatibleVersion, err)
  1817  		suggestFixes()
  1818  		return
  1819  	}
  1820  
  1821  	// Re-resolve packages in parallel.
  1822  	//
  1823  	// We re-resolve each package — rather than just checking versions — to ensure
  1824  	// that we have fetched module source code (and, importantly, checksums for
  1825  	// that source code) for all modules that are necessary to ensure that imports
  1826  	// are unambiguous. That also produces clearer diagnostics, since we can say
  1827  	// exactly what happened to the package if it became ambiguous or disappeared
  1828  	// entirely.
  1829  	//
  1830  	// We re-resolve the packages in parallel because this process involves disk
  1831  	// I/O to check for package sources, and because the process of checking for
  1832  	// ambiguous imports may require us to download additional modules that are
  1833  	// otherwise pruned out in Go 1.17 — we don't want to block progress on other
  1834  	// packages while we wait for a single new download.
  1835  	type mismatch struct {
  1836  		mod module.Version
  1837  		err error
  1838  	}
  1839  	mismatchMu := make(chan map[*loadPkg]mismatch, 1)
  1840  	mismatchMu <- map[*loadPkg]mismatch{}
  1841  	for _, pkg := range ld.pkgs {
  1842  		if pkg.mod.Path == "" && pkg.err == nil {
  1843  			// This package is from the standard library (which does not vary based on
  1844  			// the module graph).
  1845  			continue
  1846  		}
  1847  
  1848  		pkg := pkg
  1849  		ld.work.Add(func() {
  1850  			mod, _, err := importFromModules(ctx, pkg.path, rs, mg)
  1851  			if mod != pkg.mod {
  1852  				mismatches := <-mismatchMu
  1853  				mismatches[pkg] = mismatch{mod: mod, err: err}
  1854  				mismatchMu <- mismatches
  1855  			}
  1856  		})
  1857  	}
  1858  	<-ld.work.Idle()
  1859  
  1860  	mismatches := <-mismatchMu
  1861  	if len(mismatches) == 0 {
  1862  		// Since we're running as part of 'go mod tidy', the roots of the module
  1863  		// graph should contain only modules that are relevant to some package in
  1864  		// the package graph. We checked every package in the package graph and
  1865  		// didn't find any mismatches, so that must mean that all of the roots of
  1866  		// the module graph are also consistent.
  1867  		//
  1868  		// If we're wrong, Go 1.16 in -mod=readonly mode will error out with
  1869  		// "updates to go.mod needed", which would be very confusing. So instead,
  1870  		// we'll double-check that our reasoning above actually holds — if it
  1871  		// doesn't, we'll emit an internal error and hopefully the user will report
  1872  		// it as a bug.
  1873  		for _, m := range ld.requirements.rootModules {
  1874  			if v := mg.Selected(m.Path); v != m.Version {
  1875  				fmt.Fprintln(os.Stderr)
  1876  				base.Fatalf("go: internal error: failed to diagnose selected-version mismatch for module %s: go %s selects %s, but go %s selects %s\n\tPlease report this at https://golang.org/issue.", m.Path, ld.GoVersion, m.Version, ld.TidyCompatibleVersion, v)
  1877  			}
  1878  		}
  1879  		return
  1880  	}
  1881  
  1882  	// Iterate over the packages (instead of the mismatches map) to emit errors in
  1883  	// deterministic order.
  1884  	for _, pkg := range ld.pkgs {
  1885  		mismatch, ok := mismatches[pkg]
  1886  		if !ok {
  1887  			continue
  1888  		}
  1889  
  1890  		if pkg.isTest() {
  1891  			// We already did (or will) report an error for the package itself,
  1892  			// so don't report a duplicate (and more vebose) error for its test.
  1893  			if _, ok := mismatches[pkg.testOf]; !ok {
  1894  				base.Fatalf("go: internal error: mismatch recorded for test %s, but not its non-test package", pkg.path)
  1895  			}
  1896  			continue
  1897  		}
  1898  
  1899  		switch {
  1900  		case mismatch.err != nil:
  1901  			// pkg resolved successfully, but errors out using the requirements in rs.
  1902  			//
  1903  			// This could occur because the import is provided by a single lazy root
  1904  			// (and is thus unambiguous in lazy mode) and also one or more
  1905  			// transitive dependencies (and is ambiguous in eager mode).
  1906  			//
  1907  			// It could also occur because some transitive dependency upgrades the
  1908  			// module that previously provided the package to a version that no
  1909  			// longer does, or to a version for which the module source code (but
  1910  			// not the go.mod file in isolation) has a checksum error.
  1911  			if missing := (*ImportMissingError)(nil); errors.As(mismatch.err, &missing) {
  1912  				selected := module.Version{
  1913  					Path:    pkg.mod.Path,
  1914  					Version: mg.Selected(pkg.mod.Path),
  1915  				}
  1916  				ld.errorf("%s loaded from %v,\n\tbut go %s would fail to locate it in %s\n", pkg.stackText(), pkg.mod, ld.TidyCompatibleVersion, selected)
  1917  			} else {
  1918  				if ambiguous := (*AmbiguousImportError)(nil); errors.As(mismatch.err, &ambiguous) {
  1919  					// TODO: Is this check needed?
  1920  				}
  1921  				ld.errorf("%s loaded from %v,\n\tbut go %s would fail to locate it:\n\t%v\n", pkg.stackText(), pkg.mod, ld.TidyCompatibleVersion, mismatch.err)
  1922  			}
  1923  
  1924  			suggestEFlag = true
  1925  
  1926  			// Even if we press ahead with the '-e' flag, the older version will
  1927  			// error out in readonly mode if it thinks the go.mod file contains
  1928  			// any *explicit* dependency that is not at its selected version,
  1929  			// even if that dependency is not relevant to any package being loaded.
  1930  			//
  1931  			// We check for that condition here. If all of the roots are consistent
  1932  			// the '-e' flag suffices, but otherwise we need to suggest an upgrade.
  1933  			if !suggestUpgrade {
  1934  				for _, m := range ld.requirements.rootModules {
  1935  					if v := mg.Selected(m.Path); v != m.Version {
  1936  						suggestUpgrade = true
  1937  						break
  1938  					}
  1939  				}
  1940  			}
  1941  
  1942  		case pkg.err != nil:
  1943  			// pkg had an error in lazy mode (presumably suppressed with the -e flag),
  1944  			// but not in eager mode.
  1945  			//
  1946  			// This is possible, if, say, the import is unresolved in lazy mode
  1947  			// (because the "latest" version of each candidate module either is
  1948  			// unavailable or does not contain the package), but is resolved in
  1949  			// eager mode due to a newer-than-latest dependency that is normally
  1950  			// runed out of the module graph.
  1951  			//
  1952  			// This could also occur if the source code for the module providing the
  1953  			// package in lazy mode has a checksum error, but eager mode upgrades
  1954  			// that module to a version with a correct checksum.
  1955  			//
  1956  			// pkg.err should have already been logged elsewhere — along with a
  1957  			// stack trace — so log only the import path and non-error info here.
  1958  			suggestUpgrade = true
  1959  			ld.errorf("%s failed to load from any module,\n\tbut go %s would load it from %v\n", pkg.path, ld.TidyCompatibleVersion, mismatch.mod)
  1960  
  1961  		case pkg.mod != mismatch.mod:
  1962  			// The package is loaded successfully by both Go versions, but from a
  1963  			// different module in each. This could lead to subtle (and perhaps even
  1964  			// unnoticed!) variations in behavior between builds with different
  1965  			// toolchains.
  1966  			suggestUpgrade = true
  1967  			ld.errorf("%s loaded from %v,\n\tbut go %s would select %v\n", pkg.stackText(), pkg.mod, ld.TidyCompatibleVersion, mismatch.mod.Version)
  1968  
  1969  		default:
  1970  			base.Fatalf("go: internal error: mismatch recorded for package %s, but no differences found", pkg.path)
  1971  		}
  1972  	}
  1973  
  1974  	suggestFixes()
  1975  	base.ExitIfErrors()
  1976  }
  1977  
  1978  // scanDir is like imports.ScanDir but elides known magic imports from the list,
  1979  // so that we do not go looking for packages that don't really exist.
  1980  //
  1981  // The standard magic import is "C", for cgo.
  1982  //
  1983  // The only other known magic imports are appengine and appengine/*.
  1984  // These are so old that they predate "go get" and did not use URL-like paths.
  1985  // Most code today now uses google.golang.org/appengine instead,
  1986  // but not all code has been so updated. When we mostly ignore build tags
  1987  // during "go vendor", we look into "// +build appengine" files and
  1988  // may see these legacy imports. We drop them so that the module
  1989  // search does not look for modules to try to satisfy them.
  1990  func scanDir(dir string, tags map[string]bool) (imports_, testImports []string, err error) {
  1991  	imports_, testImports, err = imports.ScanDir(dir, tags)
  1992  
  1993  	filter := func(x []string) []string {
  1994  		w := 0
  1995  		for _, pkg := range x {
  1996  			if pkg != "C" && pkg != "appengine" && !strings.HasPrefix(pkg, "appengine/") &&
  1997  				pkg != "appengine_internal" && !strings.HasPrefix(pkg, "appengine_internal/") {
  1998  				x[w] = pkg
  1999  				w++
  2000  			}
  2001  		}
  2002  		return x[:w]
  2003  	}
  2004  
  2005  	return filter(imports_), filter(testImports), err
  2006  }
  2007  
  2008  // buildStacks computes minimal import stacks for each package,
  2009  // for use in error messages. When it completes, packages that
  2010  // are part of the original root set have pkg.stack == nil,
  2011  // and other packages have pkg.stack pointing at the next
  2012  // package up the import stack in their minimal chain.
  2013  // As a side effect, buildStacks also constructs ld.pkgs,
  2014  // the list of all packages loaded.
  2015  func (ld *loader) buildStacks() {
  2016  	if len(ld.pkgs) > 0 {
  2017  		panic("buildStacks")
  2018  	}
  2019  	for _, pkg := range ld.roots {
  2020  		pkg.stack = pkg // sentinel to avoid processing in next loop
  2021  		ld.pkgs = append(ld.pkgs, pkg)
  2022  	}
  2023  	for i := 0; i < len(ld.pkgs); i++ { // not range: appending to ld.pkgs in loop
  2024  		pkg := ld.pkgs[i]
  2025  		for _, next := range pkg.imports {
  2026  			if next.stack == nil {
  2027  				next.stack = pkg
  2028  				ld.pkgs = append(ld.pkgs, next)
  2029  			}
  2030  		}
  2031  		if next := pkg.test; next != nil && next.stack == nil {
  2032  			next.stack = pkg
  2033  			ld.pkgs = append(ld.pkgs, next)
  2034  		}
  2035  	}
  2036  	for _, pkg := range ld.roots {
  2037  		pkg.stack = nil
  2038  	}
  2039  }
  2040  
  2041  // stackText builds the import stack text to use when
  2042  // reporting an error in pkg. It has the general form
  2043  //
  2044  //	root imports
  2045  //		other imports
  2046  //		other2 tested by
  2047  //		other2.test imports
  2048  //		pkg
  2049  //
  2050  func (pkg *loadPkg) stackText() string {
  2051  	var stack []*loadPkg
  2052  	for p := pkg; p != nil; p = p.stack {
  2053  		stack = append(stack, p)
  2054  	}
  2055  
  2056  	var buf bytes.Buffer
  2057  	for i := len(stack) - 1; i >= 0; i-- {
  2058  		p := stack[i]
  2059  		fmt.Fprint(&buf, p.path)
  2060  		if p.testOf != nil {
  2061  			fmt.Fprint(&buf, ".test")
  2062  		}
  2063  		if i > 0 {
  2064  			if stack[i-1].testOf == p {
  2065  				fmt.Fprint(&buf, " tested by\n\t")
  2066  			} else {
  2067  				fmt.Fprint(&buf, " imports\n\t")
  2068  			}
  2069  		}
  2070  	}
  2071  	return buf.String()
  2072  }
  2073  
  2074  // why returns the text to use in "go mod why" output about the given package.
  2075  // It is less ornate than the stackText but contains the same information.
  2076  func (pkg *loadPkg) why() string {
  2077  	var buf strings.Builder
  2078  	var stack []*loadPkg
  2079  	for p := pkg; p != nil; p = p.stack {
  2080  		stack = append(stack, p)
  2081  	}
  2082  
  2083  	for i := len(stack) - 1; i >= 0; i-- {
  2084  		p := stack[i]
  2085  		if p.testOf != nil {
  2086  			fmt.Fprintf(&buf, "%s.test\n", p.testOf.path)
  2087  		} else {
  2088  			fmt.Fprintf(&buf, "%s\n", p.path)
  2089  		}
  2090  	}
  2091  	return buf.String()
  2092  }
  2093  
  2094  // Why returns the "go mod why" output stanza for the given package,
  2095  // without the leading # comment.
  2096  // The package graph must have been loaded already, usually by LoadPackages.
  2097  // If there is no reason for the package to be in the current build,
  2098  // Why returns an empty string.
  2099  func Why(path string) string {
  2100  	pkg, ok := loaded.pkgCache.Get(path).(*loadPkg)
  2101  	if !ok {
  2102  		return ""
  2103  	}
  2104  	return pkg.why()
  2105  }
  2106  
  2107  // WhyDepth returns the number of steps in the Why listing.
  2108  // If there is no reason for the package to be in the current build,
  2109  // WhyDepth returns 0.
  2110  func WhyDepth(path string) int {
  2111  	n := 0
  2112  	pkg, _ := loaded.pkgCache.Get(path).(*loadPkg)
  2113  	for p := pkg; p != nil; p = p.stack {
  2114  		n++
  2115  	}
  2116  	return n
  2117  }
  2118  

View as plain text