vendor.go

Documentation: cmd/go/internal/modcmd

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modcmd
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"errors"
    11  	"fmt"
    12  	"go/build"
    13  	"io"
    14  	"io/fs"
    15  	"os"
    16  	"path"
    17  	"path/filepath"
    18  	"sort"
    19  	"strings"
    20  
    21  	"cmd/go/internal/base"
    22  	"cmd/go/internal/cfg"
    23  	"cmd/go/internal/fsys"
    24  	"cmd/go/internal/imports"
    25  	"cmd/go/internal/load"
    26  	"cmd/go/internal/modload"
    27  	"cmd/go/internal/str"
    28  
    29  	"golang.org/x/mod/module"
    30  	"golang.org/x/mod/semver"
    31  )
    32  
    33  var cmdVendor = &base.Command{
    34  	UsageLine: "go mod vendor [-e] [-v]",
    35  	Short:     "make vendored copy of dependencies",
    36  	Long: `
    37  Vendor resets the main module's vendor directory to include all packages
    38  needed to build and test all the main module's packages.
    39  It does not include test code for vendored packages.
    40  
    41  The -v flag causes vendor to print the names of vendored
    42  modules and packages to standard error.
    43  
    44  The -e flag causes vendor to attempt to proceed despite errors
    45  encountered while loading packages.
    46  
    47  See https://golang.org/ref/mod#go-mod-vendor for more about 'go mod vendor'.
    48  	`,
    49  	Run: runVendor,
    50  }
    51  
    52  var vendorE bool // if true, report errors but proceed anyway
    53  
    54  func init() {
    55  	cmdVendor.Flag.BoolVar(&cfg.BuildV, "v", false, "")
    56  	cmdVendor.Flag.BoolVar(&vendorE, "e", false, "")
    57  	base.AddModCommonFlags(&cmdVendor.Flag)
    58  }
    59  
    60  func runVendor(ctx context.Context, cmd *base.Command, args []string) {
    61  	if len(args) != 0 {
    62  		base.Fatalf("go mod vendor: vendor takes no arguments")
    63  	}
    64  	modload.ForceUseModules = true
    65  	modload.RootMode = modload.NeedRoot
    66  
    67  	loadOpts := modload.PackageOpts{
    68  		Tags:                     imports.AnyTags(),
    69  		VendorModulesInGOROOTSrc: true,
    70  		ResolveMissingImports:    true,
    71  		UseVendorAll:             true,
    72  		AllowErrors:              vendorE,
    73  		SilenceMissingStdImports: true,
    74  	}
    75  	_, pkgs := modload.LoadPackages(ctx, loadOpts, "all")
    76  
    77  	vdir := filepath.Join(modload.ModRoot(), "vendor")
    78  	if err := os.RemoveAll(vdir); err != nil {
    79  		base.Fatalf("go mod vendor: %v", err)
    80  	}
    81  
    82  	modpkgs := make(map[module.Version][]string)
    83  	for _, pkg := range pkgs {
    84  		m := modload.PackageModule(pkg)
    85  		if m.Path == "" || m == modload.Target {
    86  			continue
    87  		}
    88  		modpkgs[m] = append(modpkgs[m], pkg)
    89  	}
    90  
    91  	includeAllReplacements := false
    92  	includeGoVersions := false
    93  	isExplicit := map[module.Version]bool{}
    94  	if gv := modload.ModFile().Go; gv != nil {
    95  		if semver.Compare("v"+gv.Version, "v1.14") >= 0 {
    96  			// If the Go version is at least 1.14, annotate all explicit 'require' and
    97  			// 'replace' targets found in the go.mod file so that we can perform a
    98  			// stronger consistency check when -mod=vendor is set.
    99  			for _, r := range modload.ModFile().Require {
   100  				isExplicit[r.Mod] = true
   101  			}
   102  			includeAllReplacements = true
   103  		}
   104  		if semver.Compare("v"+gv.Version, "v1.17") >= 0 {
   105  			// If the Go version is at least 1.17, annotate all modules with their
   106  			// 'go' version directives.
   107  			includeGoVersions = true
   108  		}
   109  	}
   110  
   111  	var vendorMods []module.Version
   112  	for m := range isExplicit {
   113  		vendorMods = append(vendorMods, m)
   114  	}
   115  	for m := range modpkgs {
   116  		if !isExplicit[m] {
   117  			vendorMods = append(vendorMods, m)
   118  		}
   119  	}
   120  	module.Sort(vendorMods)
   121  
   122  	var (
   123  		buf bytes.Buffer
   124  		w   io.Writer = &buf
   125  	)
   126  	if cfg.BuildV {
   127  		w = io.MultiWriter(&buf, os.Stderr)
   128  	}
   129  
   130  	for _, m := range vendorMods {
   131  		line := moduleLine(m, modload.Replacement(m))
   132  		io.WriteString(w, line)
   133  
   134  		goVersion := ""
   135  		if includeGoVersions {
   136  			goVersion = modload.ModuleInfo(ctx, m.Path).GoVersion
   137  		}
   138  		switch {
   139  		case isExplicit[m] && goVersion != "":
   140  			fmt.Fprintf(w, "## explicit; go %s\n", goVersion)
   141  		case isExplicit[m]:
   142  			io.WriteString(w, "## explicit\n")
   143  		case goVersion != "":
   144  			fmt.Fprintf(w, "## go %s\n", goVersion)
   145  		}
   146  
   147  		pkgs := modpkgs[m]
   148  		sort.Strings(pkgs)
   149  		for _, pkg := range pkgs {
   150  			fmt.Fprintf(w, "%s\n", pkg)
   151  			vendorPkg(vdir, pkg)
   152  		}
   153  	}
   154  
   155  	if includeAllReplacements {
   156  		// Record unused and wildcard replacements at the end of the modules.txt file:
   157  		// without access to the complete build list, the consumer of the vendor
   158  		// directory can't otherwise determine that those replacements had no effect.
   159  		for _, r := range modload.ModFile().Replace {
   160  			if len(modpkgs[r.Old]) > 0 {
   161  				// We we already recorded this replacement in the entry for the replaced
   162  				// module with the packages it provides.
   163  				continue
   164  			}
   165  
   166  			line := moduleLine(r.Old, r.New)
   167  			buf.WriteString(line)
   168  			if cfg.BuildV {
   169  				os.Stderr.WriteString(line)
   170  			}
   171  		}
   172  	}
   173  
   174  	if buf.Len() == 0 {
   175  		fmt.Fprintf(os.Stderr, "go: no dependencies to vendor\n")
   176  		return
   177  	}
   178  
   179  	if err := os.MkdirAll(vdir, 0777); err != nil {
   180  		base.Fatalf("go mod vendor: %v", err)
   181  	}
   182  
   183  	if err := os.WriteFile(filepath.Join(vdir, "modules.txt"), buf.Bytes(), 0666); err != nil {
   184  		base.Fatalf("go mod vendor: %v", err)
   185  	}
   186  }
   187  
   188  func moduleLine(m, r module.Version) string {
   189  	b := new(strings.Builder)
   190  	b.WriteString("# ")
   191  	b.WriteString(m.Path)
   192  	if m.Version != "" {
   193  		b.WriteString(" ")
   194  		b.WriteString(m.Version)
   195  	}
   196  	if r.Path != "" {
   197  		b.WriteString(" => ")
   198  		b.WriteString(r.Path)
   199  		if r.Version != "" {
   200  			b.WriteString(" ")
   201  			b.WriteString(r.Version)
   202  		}
   203  	}
   204  	b.WriteString("\n")
   205  	return b.String()
   206  }
   207  
   208  func vendorPkg(vdir, pkg string) {
   209  	// TODO(#42504): Instead of calling modload.ImportMap then build.ImportDir,
   210  	// just call load.PackagesAndErrors. To do that, we need to add a good way
   211  	// to ignore build constraints.
   212  	realPath := modload.ImportMap(pkg)
   213  	if realPath != pkg && modload.ImportMap(realPath) != "" {
   214  		fmt.Fprintf(os.Stderr, "warning: %s imported as both %s and %s; making two copies.\n", realPath, realPath, pkg)
   215  	}
   216  
   217  	copiedFiles := make(map[string]bool)
   218  	dst := filepath.Join(vdir, pkg)
   219  	src := modload.PackageDir(realPath)
   220  	if src == "" {
   221  		fmt.Fprintf(os.Stderr, "internal error: no pkg for %s -> %s\n", pkg, realPath)
   222  	}
   223  	copyDir(dst, src, matchPotentialSourceFile, copiedFiles)
   224  	if m := modload.PackageModule(realPath); m.Path != "" {
   225  		copyMetadata(m.Path, realPath, dst, src, copiedFiles)
   226  	}
   227  
   228  	ctx := build.Default
   229  	ctx.UseAllFiles = true
   230  	bp, err := ctx.ImportDir(src, build.IgnoreVendor)
   231  	// Because UseAllFiles is set on the build.Context, it's possible ta get
   232  	// a MultiplePackageError on an otherwise valid package: the package could
   233  	// have different names for GOOS=windows and GOOS=mac for example. On the
   234  	// other hand if there's a NoGoError, the package might have source files
   235  	// specifying "// +build ignore" those packages should be skipped because
   236  	// embeds from ignored files can't be used.
   237  	// TODO(#42504): Find a better way to avoid errors from ImportDir. We'll
   238  	// need to figure this out when we switch to PackagesAndErrors as per the
   239  	// TODO above.
   240  	var multiplePackageError *build.MultiplePackageError
   241  	var noGoError *build.NoGoError
   242  	if err != nil {
   243  		if errors.As(err, &noGoError) {
   244  			return // No source files in this package are built. Skip embeds in ignored files.
   245  		} else if !errors.As(err, &multiplePackageError) { // multiplePackgeErrors are okay, but others are not.
   246  			base.Fatalf("internal error: failed to find embedded files of %s: %v\n", pkg, err)
   247  		}
   248  	}
   249  	embedPatterns := str.StringList(bp.EmbedPatterns, bp.TestEmbedPatterns, bp.XTestEmbedPatterns)
   250  	embeds, err := load.ResolveEmbed(bp.Dir, embedPatterns)
   251  	if err != nil {
   252  		base.Fatalf("go mod vendor: %v", err)
   253  	}
   254  	for _, embed := range embeds {
   255  		embedDst := filepath.Join(dst, embed)
   256  		if copiedFiles[embedDst] {
   257  			continue
   258  		}
   259  
   260  		// Copy the file as is done by copyDir below.
   261  		r, err := os.Open(filepath.Join(src, embed))
   262  		if err != nil {
   263  			base.Fatalf("go mod vendor: %v", err)
   264  		}
   265  		if err := os.MkdirAll(filepath.Dir(embedDst), 0777); err != nil {
   266  			base.Fatalf("go mod vendor: %v", err)
   267  		}
   268  		w, err := os.Create(embedDst)
   269  		if err != nil {
   270  			base.Fatalf("go mod vendor: %v", err)
   271  		}
   272  		if _, err := io.Copy(w, r); err != nil {
   273  			base.Fatalf("go mod vendor: %v", err)
   274  		}
   275  		r.Close()
   276  		if err := w.Close(); err != nil {
   277  			base.Fatalf("go mod vendor: %v", err)
   278  		}
   279  	}
   280  }
   281  
   282  type metakey struct {
   283  	modPath string
   284  	dst     string
   285  }
   286  
   287  var copiedMetadata = make(map[metakey]bool)
   288  
   289  // copyMetadata copies metadata files from parents of src to parents of dst,
   290  // stopping after processing the src parent for modPath.
   291  func copyMetadata(modPath, pkg, dst, src string, copiedFiles map[string]bool) {
   292  	for parent := 0; ; parent++ {
   293  		if copiedMetadata[metakey{modPath, dst}] {
   294  			break
   295  		}
   296  		copiedMetadata[metakey{modPath, dst}] = true
   297  		if parent > 0 {
   298  			copyDir(dst, src, matchMetadata, copiedFiles)
   299  		}
   300  		if modPath == pkg {
   301  			break
   302  		}
   303  		pkg = path.Dir(pkg)
   304  		dst = filepath.Dir(dst)
   305  		src = filepath.Dir(src)
   306  	}
   307  }
   308  
   309  // metaPrefixes is the list of metadata file prefixes.
   310  // Vendoring copies metadata files from parents of copied directories.
   311  // Note that this list could be arbitrarily extended, and it is longer
   312  // in other tools (such as godep or dep). By using this limited set of
   313  // prefixes and also insisting on capitalized file names, we are trying
   314  // to nudge people toward more agreement on the naming
   315  // and also trying to avoid false positives.
   316  var metaPrefixes = []string{
   317  	"AUTHORS",
   318  	"CONTRIBUTORS",
   319  	"COPYLEFT",
   320  	"COPYING",
   321  	"COPYRIGHT",
   322  	"LEGAL",
   323  	"LICENSE",
   324  	"NOTICE",
   325  	"PATENTS",
   326  }
   327  
   328  // matchMetadata reports whether info is a metadata file.
   329  func matchMetadata(dir string, info fs.DirEntry) bool {
   330  	name := info.Name()
   331  	for _, p := range metaPrefixes {
   332  		if strings.HasPrefix(name, p) {
   333  			return true
   334  		}
   335  	}
   336  	return false
   337  }
   338  
   339  // matchPotentialSourceFile reports whether info may be relevant to a build operation.
   340  func matchPotentialSourceFile(dir string, info fs.DirEntry) bool {
   341  	if strings.HasSuffix(info.Name(), "_test.go") {
   342  		return false
   343  	}
   344  	if info.Name() == "go.mod" || info.Name() == "go.sum" {
   345  		if gv := modload.ModFile().Go; gv != nil && semver.Compare("v"+gv.Version, "v1.17") >= 0 {
   346  			// As of Go 1.17, we strip go.mod and go.sum files from dependency modules.
   347  			// Otherwise, 'go' commands invoked within the vendor subtree may misidentify
   348  			// an arbitrary directory within the vendor tree as a module root.
   349  			// (See https://golang.org/issue/42970.)
   350  			return false
   351  		}
   352  	}
   353  	if strings.HasSuffix(info.Name(), ".go") {
   354  		f, err := fsys.Open(filepath.Join(dir, info.Name()))
   355  		if err != nil {
   356  			base.Fatalf("go mod vendor: %v", err)
   357  		}
   358  		defer f.Close()
   359  
   360  		content, err := imports.ReadImports(f, false, nil)
   361  		if err == nil && !imports.ShouldBuild(content, imports.AnyTags()) {
   362  			// The file is explicitly tagged "ignore", so it can't affect the build.
   363  			// Leave it out.
   364  			return false
   365  		}
   366  		return true
   367  	}
   368  
   369  	// We don't know anything about this file, so optimistically assume that it is
   370  	// needed.
   371  	return true
   372  }
   373  
   374  // copyDir copies all regular files satisfying match(info) from src to dst.
   375  func copyDir(dst, src string, match func(dir string, info fs.DirEntry) bool, copiedFiles map[string]bool) {
   376  	files, err := os.ReadDir(src)
   377  	if err != nil {
   378  		base.Fatalf("go mod vendor: %v", err)
   379  	}
   380  	if err := os.MkdirAll(dst, 0777); err != nil {
   381  		base.Fatalf("go mod vendor: %v", err)
   382  	}
   383  	for _, file := range files {
   384  		if file.IsDir() || !file.Type().IsRegular() || !match(src, file) {
   385  			continue
   386  		}
   387  		copiedFiles[file.Name()] = true
   388  		r, err := os.Open(filepath.Join(src, file.Name()))
   389  		if err != nil {
   390  			base.Fatalf("go mod vendor: %v", err)
   391  		}
   392  		dstPath := filepath.Join(dst, file.Name())
   393  		copiedFiles[dstPath] = true
   394  		w, err := os.Create(dstPath)
   395  		if err != nil {
   396  			base.Fatalf("go mod vendor: %v", err)
   397  		}
   398  		if _, err := io.Copy(w, r); err != nil {
   399  			base.Fatalf("go mod vendor: %v", err)
   400  		}
   401  		r.Close()
   402  		if err := w.Close(); err != nil {
   403  			base.Fatalf("go mod vendor: %v", err)
   404  		}
   405  	}
   406  }
   407
View as plain text