Source file src/cmd/go/internal/modfetch/fetch.go

Documentation: cmd/go/internal/modfetch

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modfetch
     6  
     7  import (
     8  	"archive/zip"
     9  	"bytes"
    10  	"context"
    11  	"crypto/sha256"
    12  	"encoding/base64"
    13  	"errors"
    14  	"fmt"
    15  	"io"
    16  	"io/fs"
    17  	"os"
    18  	"path/filepath"
    19  	"sort"
    20  	"strings"
    21  	"sync"
    22  
    23  	"cmd/go/internal/base"
    24  	"cmd/go/internal/cfg"
    25  	"cmd/go/internal/fsys"
    26  	"cmd/go/internal/lockedfile"
    27  	"cmd/go/internal/par"
    28  	"cmd/go/internal/robustio"
    29  	"cmd/go/internal/trace"
    30  
    31  	"golang.org/x/mod/module"
    32  	"golang.org/x/mod/sumdb/dirhash"
    33  	modzip "golang.org/x/mod/zip"
    34  )
    35  
    36  var downloadCache par.Cache
    37  
    38  // Download downloads the specific module version to the
    39  // local download cache and returns the name of the directory
    40  // corresponding to the root of the module's file tree.
    41  func Download(ctx context.Context, mod module.Version) (dir string, err error) {
    42  	if err := checkCacheDir(); err != nil {
    43  		base.Fatalf("go: %v", err)
    44  	}
    45  
    46  	// The par.Cache here avoids duplicate work.
    47  	type cached struct {
    48  		dir string
    49  		err error
    50  	}
    51  	c := downloadCache.Do(mod, func() interface{} {
    52  		dir, err := download(ctx, mod)
    53  		if err != nil {
    54  			return cached{"", err}
    55  		}
    56  		checkMod(mod)
    57  		return cached{dir, nil}
    58  	}).(cached)
    59  	return c.dir, c.err
    60  }
    61  
    62  func download(ctx context.Context, mod module.Version) (dir string, err error) {
    63  	ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String())
    64  	defer span.Done()
    65  
    66  	dir, err = DownloadDir(mod)
    67  	if err == nil {
    68  		// The directory has already been completely extracted (no .partial file exists).
    69  		return dir, nil
    70  	} else if dir == "" || !errors.Is(err, fs.ErrNotExist) {
    71  		return "", err
    72  	}
    73  
    74  	// To avoid cluttering the cache with extraneous files,
    75  	// DownloadZip uses the same lockfile as Download.
    76  	// Invoke DownloadZip before locking the file.
    77  	zipfile, err := DownloadZip(ctx, mod)
    78  	if err != nil {
    79  		return "", err
    80  	}
    81  
    82  	unlock, err := lockVersion(mod)
    83  	if err != nil {
    84  		return "", err
    85  	}
    86  	defer unlock()
    87  
    88  	ctx, span = trace.StartSpan(ctx, "unzip "+zipfile)
    89  	defer span.Done()
    90  
    91  	// Check whether the directory was populated while we were waiting on the lock.
    92  	_, dirErr := DownloadDir(mod)
    93  	if dirErr == nil {
    94  		return dir, nil
    95  	}
    96  	_, dirExists := dirErr.(*DownloadDirPartialError)
    97  
    98  	// Clean up any remaining temporary directories created by old versions
    99  	// (before 1.16), as well as partially extracted directories (indicated by
   100  	// DownloadDirPartialError, usually because of a .partial file). This is only
   101  	// safe to do because the lock file ensures that their writers are no longer
   102  	// active.
   103  	parentDir := filepath.Dir(dir)
   104  	tmpPrefix := filepath.Base(dir) + ".tmp-"
   105  	if old, err := filepath.Glob(filepath.Join(parentDir, tmpPrefix+"*")); err == nil {
   106  		for _, path := range old {
   107  			RemoveAll(path) // best effort
   108  		}
   109  	}
   110  	if dirExists {
   111  		if err := RemoveAll(dir); err != nil {
   112  			return "", err
   113  		}
   114  	}
   115  
   116  	partialPath, err := CachePath(mod, "partial")
   117  	if err != nil {
   118  		return "", err
   119  	}
   120  
   121  	// Extract the module zip directory at its final location.
   122  	//
   123  	// To prevent other processes from reading the directory if we crash,
   124  	// create a .partial file before extracting the directory, and delete
   125  	// the .partial file afterward (all while holding the lock).
   126  	//
   127  	// Before Go 1.16, we extracted to a temporary directory with a random name
   128  	// then renamed it into place with os.Rename. On Windows, this failed with
   129  	// ERROR_ACCESS_DENIED when another process (usually an anti-virus scanner)
   130  	// opened files in the temporary directory.
   131  	//
   132  	// Go 1.14.2 and higher respect .partial files. Older versions may use
   133  	// partially extracted directories. 'go mod verify' can detect this,
   134  	// and 'go clean -modcache' can fix it.
   135  	if err := os.MkdirAll(parentDir, 0777); err != nil {
   136  		return "", err
   137  	}
   138  	if err := os.WriteFile(partialPath, nil, 0666); err != nil {
   139  		return "", err
   140  	}
   141  	if err := modzip.Unzip(dir, mod, zipfile); err != nil {
   142  		fmt.Fprintf(os.Stderr, "-> %s\n", err)
   143  		if rmErr := RemoveAll(dir); rmErr == nil {
   144  			os.Remove(partialPath)
   145  		}
   146  		return "", err
   147  	}
   148  	if err := os.Remove(partialPath); err != nil {
   149  		return "", err
   150  	}
   151  
   152  	if !cfg.ModCacheRW {
   153  		makeDirsReadOnly(dir)
   154  	}
   155  	return dir, nil
   156  }
   157  
   158  var downloadZipCache par.Cache
   159  
   160  // DownloadZip downloads the specific module version to the
   161  // local zip cache and returns the name of the zip file.
   162  func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err error) {
   163  	// The par.Cache here avoids duplicate work.
   164  	type cached struct {
   165  		zipfile string
   166  		err     error
   167  	}
   168  	c := downloadZipCache.Do(mod, func() interface{} {
   169  		zipfile, err := CachePath(mod, "zip")
   170  		if err != nil {
   171  			return cached{"", err}
   172  		}
   173  		ziphashfile := zipfile + "hash"
   174  
   175  		// Return without locking if the zip and ziphash files exist.
   176  		if _, err := os.Stat(zipfile); err == nil {
   177  			if _, err := os.Stat(ziphashfile); err == nil {
   178  				return cached{zipfile, nil}
   179  			}
   180  		}
   181  
   182  		// The zip or ziphash file does not exist. Acquire the lock and create them.
   183  		if cfg.CmdName != "mod download" {
   184  			fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, mod.Version)
   185  		}
   186  		unlock, err := lockVersion(mod)
   187  		if err != nil {
   188  			return cached{"", err}
   189  		}
   190  		defer unlock()
   191  
   192  		if err := downloadZip(ctx, mod, zipfile); err != nil {
   193  			return cached{"", err}
   194  		}
   195  		return cached{zipfile, nil}
   196  	}).(cached)
   197  	return c.zipfile, c.err
   198  }
   199  
   200  func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err error) {
   201  	ctx, span := trace.StartSpan(ctx, "modfetch.downloadZip "+zipfile)
   202  	defer span.Done()
   203  
   204  	// Double-check that the zipfile was not created while we were waiting for
   205  	// the lock in DownloadZip.
   206  	ziphashfile := zipfile + "hash"
   207  	var zipExists, ziphashExists bool
   208  	if _, err := os.Stat(zipfile); err == nil {
   209  		zipExists = true
   210  	}
   211  	if _, err := os.Stat(ziphashfile); err == nil {
   212  		ziphashExists = true
   213  	}
   214  	if zipExists && ziphashExists {
   215  		return nil
   216  	}
   217  
   218  	// Create parent directories.
   219  	if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil {
   220  		return err
   221  	}
   222  
   223  	// Clean up any remaining tempfiles from previous runs.
   224  	// This is only safe to do because the lock file ensures that their
   225  	// writers are no longer active.
   226  	tmpPattern := filepath.Base(zipfile) + "*.tmp"
   227  	if old, err := filepath.Glob(filepath.Join(filepath.Dir(zipfile), tmpPattern)); err == nil {
   228  		for _, path := range old {
   229  			os.Remove(path) // best effort
   230  		}
   231  	}
   232  
   233  	// If the zip file exists, the ziphash file must have been deleted
   234  	// or lost after a file system crash. Re-hash the zip without downloading.
   235  	if zipExists {
   236  		return hashZip(mod, zipfile, ziphashfile)
   237  	}
   238  
   239  	// From here to the os.Rename call below is functionally almost equivalent to
   240  	// renameio.WriteToFile, with one key difference: we want to validate the
   241  	// contents of the file (by hashing it) before we commit it. Because the file
   242  	// is zip-compressed, we need an actual file — or at least an io.ReaderAt — to
   243  	// validate it: we can't just tee the stream as we write it.
   244  	f, err := os.CreateTemp(filepath.Dir(zipfile), tmpPattern)
   245  	if err != nil {
   246  		return err
   247  	}
   248  	defer func() {
   249  		if err != nil {
   250  			f.Close()
   251  			os.Remove(f.Name())
   252  		}
   253  	}()
   254  
   255  	var unrecoverableErr error
   256  	err = TryProxies(func(proxy string) error {
   257  		if unrecoverableErr != nil {
   258  			return unrecoverableErr
   259  		}
   260  		repo := Lookup(proxy, mod.Path)
   261  		err := repo.Zip(f, mod.Version)
   262  		if err != nil {
   263  			// Zip may have partially written to f before failing.
   264  			// (Perhaps the server crashed while sending the file?)
   265  			// Since we allow fallback on error in some cases, we need to fix up the
   266  			// file to be empty again for the next attempt.
   267  			if _, err := f.Seek(0, io.SeekStart); err != nil {
   268  				unrecoverableErr = err
   269  				return err
   270  			}
   271  			if err := f.Truncate(0); err != nil {
   272  				unrecoverableErr = err
   273  				return err
   274  			}
   275  		}
   276  		return err
   277  	})
   278  	if err != nil {
   279  		return err
   280  	}
   281  
   282  	// Double-check that the paths within the zip file are well-formed.
   283  	//
   284  	// TODO(bcmills): There is a similar check within the Unzip function. Can we eliminate one?
   285  	fi, err := f.Stat()
   286  	if err != nil {
   287  		return err
   288  	}
   289  	z, err := zip.NewReader(f, fi.Size())
   290  	if err != nil {
   291  		return err
   292  	}
   293  	prefix := mod.Path + "@" + mod.Version + "/"
   294  	for _, f := range z.File {
   295  		if !strings.HasPrefix(f.Name, prefix) {
   296  			return fmt.Errorf("zip for %s has unexpected file %s", prefix[:len(prefix)-1], f.Name)
   297  		}
   298  	}
   299  
   300  	if err := f.Close(); err != nil {
   301  		return err
   302  	}
   303  
   304  	// Hash the zip file and check the sum before renaming to the final location.
   305  	if err := hashZip(mod, f.Name(), ziphashfile); err != nil {
   306  		return err
   307  	}
   308  	if err := os.Rename(f.Name(), zipfile); err != nil {
   309  		return err
   310  	}
   311  
   312  	// TODO(bcmills): Should we make the .zip and .ziphash files read-only to discourage tampering?
   313  
   314  	return nil
   315  }
   316  
   317  // hashZip reads the zip file opened in f, then writes the hash to ziphashfile,
   318  // overwriting that file if it exists.
   319  //
   320  // If the hash does not match go.sum (or the sumdb if enabled), hashZip returns
   321  // an error and does not write ziphashfile.
   322  func hashZip(mod module.Version, zipfile, ziphashfile string) error {
   323  	hash, err := dirhash.HashZip(zipfile, dirhash.DefaultHash)
   324  	if err != nil {
   325  		return err
   326  	}
   327  	if err := checkModSum(mod, hash); err != nil {
   328  		return err
   329  	}
   330  	hf, err := lockedfile.Create(ziphashfile)
   331  	if err != nil {
   332  		return err
   333  	}
   334  	if err := hf.Truncate(int64(len(hash))); err != nil {
   335  		return err
   336  	}
   337  	if _, err := hf.WriteAt([]byte(hash), 0); err != nil {
   338  		return err
   339  	}
   340  	if err := hf.Close(); err != nil {
   341  		return err
   342  	}
   343  
   344  	return nil
   345  }
   346  
   347  // makeDirsReadOnly makes a best-effort attempt to remove write permissions for dir
   348  // and its transitive contents.
   349  func makeDirsReadOnly(dir string) {
   350  	type pathMode struct {
   351  		path string
   352  		mode fs.FileMode
   353  	}
   354  	var dirs []pathMode // in lexical order
   355  	filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
   356  		if err == nil && d.IsDir() {
   357  			info, err := d.Info()
   358  			if err == nil && info.Mode()&0222 != 0 {
   359  				dirs = append(dirs, pathMode{path, info.Mode()})
   360  			}
   361  		}
   362  		return nil
   363  	})
   364  
   365  	// Run over list backward to chmod children before parents.
   366  	for i := len(dirs) - 1; i >= 0; i-- {
   367  		os.Chmod(dirs[i].path, dirs[i].mode&^0222)
   368  	}
   369  }
   370  
   371  // RemoveAll removes a directory written by Download or Unzip, first applying
   372  // any permission changes needed to do so.
   373  func RemoveAll(dir string) error {
   374  	// Module cache has 0555 directories; make them writable in order to remove content.
   375  	filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error {
   376  		if err != nil {
   377  			return nil // ignore errors walking in file system
   378  		}
   379  		if info.IsDir() {
   380  			os.Chmod(path, 0777)
   381  		}
   382  		return nil
   383  	})
   384  	return robustio.RemoveAll(dir)
   385  }
   386  
   387  var GoSumFile string // path to go.sum; set by package modload
   388  
   389  type modSum struct {
   390  	mod module.Version
   391  	sum string
   392  }
   393  
   394  var goSum struct {
   395  	mu        sync.Mutex
   396  	m         map[module.Version][]string // content of go.sum file
   397  	status    map[modSum]modSumStatus     // state of sums in m
   398  	overwrite bool                        // if true, overwrite go.sum without incorporating its contents
   399  	enabled   bool                        // whether to use go.sum at all
   400  }
   401  
   402  type modSumStatus struct {
   403  	used, dirty bool
   404  }
   405  
   406  // initGoSum initializes the go.sum data.
   407  // The boolean it returns reports whether the
   408  // use of go.sum is now enabled.
   409  // The goSum lock must be held.
   410  func initGoSum() (bool, error) {
   411  	if GoSumFile == "" {
   412  		return false, nil
   413  	}
   414  	if goSum.m != nil {
   415  		return true, nil
   416  	}
   417  
   418  	goSum.m = make(map[module.Version][]string)
   419  	goSum.status = make(map[modSum]modSumStatus)
   420  	var (
   421  		data []byte
   422  		err  error
   423  	)
   424  	if actualSumFile, ok := fsys.OverlayPath(GoSumFile); ok {
   425  		// Don't lock go.sum if it's part of the overlay.
   426  		// On Plan 9, locking requires chmod, and we don't want to modify any file
   427  		// in the overlay. See #44700.
   428  		data, err = os.ReadFile(actualSumFile)
   429  	} else {
   430  		data, err = lockedfile.Read(GoSumFile)
   431  	}
   432  	if err != nil && !os.IsNotExist(err) {
   433  		return false, err
   434  	}
   435  	goSum.enabled = true
   436  	readGoSum(goSum.m, GoSumFile, data)
   437  
   438  	return true, nil
   439  }
   440  
   441  // emptyGoModHash is the hash of a 1-file tree containing a 0-length go.mod.
   442  // A bug caused us to write these into go.sum files for non-modules.
   443  // We detect and remove them.
   444  const emptyGoModHash = "h1:G7mAYYxgmS0lVkHyy2hEOLQCFB0DlQFTMLWggykrydY="
   445  
   446  // readGoSum parses data, which is the content of file,
   447  // and adds it to goSum.m. The goSum lock must be held.
   448  func readGoSum(dst map[module.Version][]string, file string, data []byte) error {
   449  	lineno := 0
   450  	for len(data) > 0 {
   451  		var line []byte
   452  		lineno++
   453  		i := bytes.IndexByte(data, '\n')
   454  		if i < 0 {
   455  			line, data = data, nil
   456  		} else {
   457  			line, data = data[:i], data[i+1:]
   458  		}
   459  		f := strings.Fields(string(line))
   460  		if len(f) == 0 {
   461  			// blank line; skip it
   462  			continue
   463  		}
   464  		if len(f) != 3 {
   465  			return fmt.Errorf("malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f))
   466  		}
   467  		if f[2] == emptyGoModHash {
   468  			// Old bug; drop it.
   469  			continue
   470  		}
   471  		mod := module.Version{Path: f[0], Version: f[1]}
   472  		dst[mod] = append(dst[mod], f[2])
   473  	}
   474  	return nil
   475  }
   476  
   477  // HaveSum returns true if the go.sum file contains an entry for mod.
   478  // The entry's hash must be generated with a known hash algorithm.
   479  // mod.Version may have a "/go.mod" suffix to distinguish sums for
   480  // .mod and .zip files.
   481  func HaveSum(mod module.Version) bool {
   482  	goSum.mu.Lock()
   483  	defer goSum.mu.Unlock()
   484  	inited, err := initGoSum()
   485  	if err != nil || !inited {
   486  		return false
   487  	}
   488  	for _, h := range goSum.m[mod] {
   489  		if !strings.HasPrefix(h, "h1:") {
   490  			continue
   491  		}
   492  		if !goSum.status[modSum{mod, h}].dirty {
   493  			return true
   494  		}
   495  	}
   496  	return false
   497  }
   498  
   499  // checkMod checks the given module's checksum.
   500  func checkMod(mod module.Version) {
   501  	// Do the file I/O before acquiring the go.sum lock.
   502  	ziphash, err := CachePath(mod, "ziphash")
   503  	if err != nil {
   504  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   505  	}
   506  	data, err := lockedfile.Read(ziphash)
   507  	if err != nil {
   508  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   509  	}
   510  	data = bytes.TrimSpace(data)
   511  	if !isValidSum(data) {
   512  		// Recreate ziphash file from zip file and use that to check the mod sum.
   513  		zip, err := CachePath(mod, "zip")
   514  		if err != nil {
   515  			base.Fatalf("verifying %v", module.VersionError(mod, err))
   516  		}
   517  		err = hashZip(mod, zip, ziphash)
   518  		if err != nil {
   519  			base.Fatalf("verifying %v", module.VersionError(mod, err))
   520  		}
   521  		return
   522  	}
   523  	h := string(data)
   524  	if !strings.HasPrefix(h, "h1:") {
   525  		base.Fatalf("verifying %v", module.VersionError(mod, fmt.Errorf("unexpected ziphash: %q", h)))
   526  	}
   527  
   528  	if err := checkModSum(mod, h); err != nil {
   529  		base.Fatalf("%s", err)
   530  	}
   531  }
   532  
   533  // goModSum returns the checksum for the go.mod contents.
   534  func goModSum(data []byte) (string, error) {
   535  	return dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) {
   536  		return io.NopCloser(bytes.NewReader(data)), nil
   537  	})
   538  }
   539  
   540  // checkGoMod checks the given module's go.mod checksum;
   541  // data is the go.mod content.
   542  func checkGoMod(path, version string, data []byte) error {
   543  	h, err := goModSum(data)
   544  	if err != nil {
   545  		return &module.ModuleError{Path: path, Version: version, Err: fmt.Errorf("verifying go.mod: %v", err)}
   546  	}
   547  
   548  	return checkModSum(module.Version{Path: path, Version: version + "/go.mod"}, h)
   549  }
   550  
   551  // checkModSum checks that the recorded checksum for mod is h.
   552  //
   553  // mod.Version may have the additional suffix "/go.mod" to request the checksum
   554  // for the module's go.mod file only.
   555  func checkModSum(mod module.Version, h string) error {
   556  	// We lock goSum when manipulating it,
   557  	// but we arrange to release the lock when calling checkSumDB,
   558  	// so that parallel calls to checkModHash can execute parallel calls
   559  	// to checkSumDB.
   560  
   561  	// Check whether mod+h is listed in go.sum already. If so, we're done.
   562  	goSum.mu.Lock()
   563  	inited, err := initGoSum()
   564  	if err != nil {
   565  		goSum.mu.Unlock()
   566  		return err
   567  	}
   568  	done := inited && haveModSumLocked(mod, h)
   569  	if inited {
   570  		st := goSum.status[modSum{mod, h}]
   571  		st.used = true
   572  		goSum.status[modSum{mod, h}] = st
   573  	}
   574  	goSum.mu.Unlock()
   575  
   576  	if done {
   577  		return nil
   578  	}
   579  
   580  	// Not listed, so we want to add them.
   581  	// Consult checksum database if appropriate.
   582  	if useSumDB(mod) {
   583  		// Calls base.Fatalf if mismatch detected.
   584  		if err := checkSumDB(mod, h); err != nil {
   585  			return err
   586  		}
   587  	}
   588  
   589  	// Add mod+h to go.sum, if it hasn't appeared already.
   590  	if inited {
   591  		goSum.mu.Lock()
   592  		addModSumLocked(mod, h)
   593  		st := goSum.status[modSum{mod, h}]
   594  		st.dirty = true
   595  		goSum.status[modSum{mod, h}] = st
   596  		goSum.mu.Unlock()
   597  	}
   598  	return nil
   599  }
   600  
   601  // haveModSumLocked reports whether the pair mod,h is already listed in go.sum.
   602  // If it finds a conflicting pair instead, it calls base.Fatalf.
   603  // goSum.mu must be locked.
   604  func haveModSumLocked(mod module.Version, h string) bool {
   605  	for _, vh := range goSum.m[mod] {
   606  		if h == vh {
   607  			return true
   608  		}
   609  		if strings.HasPrefix(vh, "h1:") {
   610  			base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\tgo.sum:     %v"+goSumMismatch, mod.Path, mod.Version, h, vh)
   611  		}
   612  	}
   613  	return false
   614  }
   615  
   616  // addModSumLocked adds the pair mod,h to go.sum.
   617  // goSum.mu must be locked.
   618  func addModSumLocked(mod module.Version, h string) {
   619  	if haveModSumLocked(mod, h) {
   620  		return
   621  	}
   622  	if len(goSum.m[mod]) > 0 {
   623  		fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v"+hashVersionMismatch, mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h)
   624  	}
   625  	goSum.m[mod] = append(goSum.m[mod], h)
   626  }
   627  
   628  // checkSumDB checks the mod, h pair against the Go checksum database.
   629  // It calls base.Fatalf if the hash is to be rejected.
   630  func checkSumDB(mod module.Version, h string) error {
   631  	modWithoutSuffix := mod
   632  	noun := "module"
   633  	if strings.HasSuffix(mod.Version, "/go.mod") {
   634  		noun = "go.mod"
   635  		modWithoutSuffix.Version = strings.TrimSuffix(mod.Version, "/go.mod")
   636  	}
   637  
   638  	db, lines, err := lookupSumDB(mod)
   639  	if err != nil {
   640  		return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: %v", noun, err))
   641  	}
   642  
   643  	have := mod.Path + " " + mod.Version + " " + h
   644  	prefix := mod.Path + " " + mod.Version + " h1:"
   645  	for _, line := range lines {
   646  		if line == have {
   647  			return nil
   648  		}
   649  		if strings.HasPrefix(line, prefix) {
   650  			return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, noun, h, db, line[len(prefix)-len("h1:"):]))
   651  		}
   652  	}
   653  	return nil
   654  }
   655  
   656  // Sum returns the checksum for the downloaded copy of the given module,
   657  // if present in the download cache.
   658  func Sum(mod module.Version) string {
   659  	if cfg.GOMODCACHE == "" {
   660  		// Do not use current directory.
   661  		return ""
   662  	}
   663  
   664  	ziphash, err := CachePath(mod, "ziphash")
   665  	if err != nil {
   666  		return ""
   667  	}
   668  	data, err := lockedfile.Read(ziphash)
   669  	if err != nil {
   670  		return ""
   671  	}
   672  	data = bytes.TrimSpace(data)
   673  	if !isValidSum(data) {
   674  		return ""
   675  	}
   676  	return string(data)
   677  }
   678  
   679  // isValidSum returns true if data is the valid contents of a zip hash file.
   680  // Certain critical files are written to disk by first truncating
   681  // then writing the actual bytes, so that if the write fails
   682  // the corrupt file should contain at least one of the null
   683  // bytes written by the truncate operation.
   684  func isValidSum(data []byte) bool {
   685  	if bytes.IndexByte(data, '\000') >= 0 {
   686  		return false
   687  	}
   688  
   689  	if len(data) != len("h1:")+base64.StdEncoding.EncodedLen(sha256.Size) {
   690  		return false
   691  	}
   692  
   693  	return true
   694  }
   695  
   696  // WriteGoSum writes the go.sum file if it needs to be updated.
   697  //
   698  // keep is used to check whether a newly added sum should be saved in go.sum.
   699  // It should have entries for both module content sums and go.mod sums
   700  // (version ends with "/go.mod"). Existing sums will be preserved unless they
   701  // have been marked for deletion with TrimGoSum.
   702  func WriteGoSum(keep map[module.Version]bool) {
   703  	goSum.mu.Lock()
   704  	defer goSum.mu.Unlock()
   705  
   706  	// If we haven't read the go.sum file yet, don't bother writing it.
   707  	if !goSum.enabled {
   708  		return
   709  	}
   710  
   711  	// Check whether we need to add sums for which keep[m] is true or remove
   712  	// unused sums marked with TrimGoSum. If there are no changes to make,
   713  	// just return without opening go.sum.
   714  	dirty := false
   715  Outer:
   716  	for m, hs := range goSum.m {
   717  		for _, h := range hs {
   718  			st := goSum.status[modSum{m, h}]
   719  			if st.dirty && (!st.used || keep[m]) {
   720  				dirty = true
   721  				break Outer
   722  			}
   723  		}
   724  	}
   725  	if !dirty {
   726  		return
   727  	}
   728  	if cfg.BuildMod == "readonly" {
   729  		base.Fatalf("go: updates to go.sum needed, disabled by -mod=readonly")
   730  	}
   731  	if _, ok := fsys.OverlayPath(GoSumFile); ok {
   732  		base.Fatalf("go: updates to go.sum needed, but go.sum is part of the overlay specified with -overlay")
   733  	}
   734  
   735  	// Make a best-effort attempt to acquire the side lock, only to exclude
   736  	// previous versions of the 'go' command from making simultaneous edits.
   737  	if unlock, err := SideLock(); err == nil {
   738  		defer unlock()
   739  	}
   740  
   741  	err := lockedfile.Transform(GoSumFile, func(data []byte) ([]byte, error) {
   742  		if !goSum.overwrite {
   743  			// Incorporate any sums added by other processes in the meantime.
   744  			// Add only the sums that we actually checked: the user may have edited or
   745  			// truncated the file to remove erroneous hashes, and we shouldn't restore
   746  			// them without good reason.
   747  			goSum.m = make(map[module.Version][]string, len(goSum.m))
   748  			readGoSum(goSum.m, GoSumFile, data)
   749  			for ms, st := range goSum.status {
   750  				if st.used {
   751  					addModSumLocked(ms.mod, ms.sum)
   752  				}
   753  			}
   754  		}
   755  
   756  		var mods []module.Version
   757  		for m := range goSum.m {
   758  			mods = append(mods, m)
   759  		}
   760  		module.Sort(mods)
   761  
   762  		var buf bytes.Buffer
   763  		for _, m := range mods {
   764  			list := goSum.m[m]
   765  			sort.Strings(list)
   766  			for _, h := range list {
   767  				st := goSum.status[modSum{m, h}]
   768  				if !st.dirty || (st.used && keep[m]) {
   769  					fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h)
   770  				}
   771  			}
   772  		}
   773  		return buf.Bytes(), nil
   774  	})
   775  
   776  	if err != nil {
   777  		base.Fatalf("go: updating go.sum: %v", err)
   778  	}
   779  
   780  	goSum.status = make(map[modSum]modSumStatus)
   781  	goSum.overwrite = false
   782  }
   783  
   784  // TrimGoSum trims go.sum to contain only the modules needed for reproducible
   785  // builds.
   786  //
   787  // keep is used to check whether a sum should be retained in go.mod. It should
   788  // have entries for both module content sums and go.mod sums (version ends
   789  // with "/go.mod").
   790  func TrimGoSum(keep map[module.Version]bool) {
   791  	goSum.mu.Lock()
   792  	defer goSum.mu.Unlock()
   793  	inited, err := initGoSum()
   794  	if err != nil {
   795  		base.Fatalf("%s", err)
   796  	}
   797  	if !inited {
   798  		return
   799  	}
   800  
   801  	for m, hs := range goSum.m {
   802  		if !keep[m] {
   803  			for _, h := range hs {
   804  				goSum.status[modSum{m, h}] = modSumStatus{used: false, dirty: true}
   805  			}
   806  			goSum.overwrite = true
   807  		}
   808  	}
   809  }
   810  
   811  const goSumMismatch = `
   812  
   813  SECURITY ERROR
   814  This download does NOT match an earlier download recorded in go.sum.
   815  The bits may have been replaced on the origin server, or an attacker may
   816  have intercepted the download attempt.
   817  
   818  For more information, see 'go help module-auth'.
   819  `
   820  
   821  const sumdbMismatch = `
   822  
   823  SECURITY ERROR
   824  This download does NOT match the one reported by the checksum server.
   825  The bits may have been replaced on the origin server, or an attacker may
   826  have intercepted the download attempt.
   827  
   828  For more information, see 'go help module-auth'.
   829  `
   830  
   831  const hashVersionMismatch = `
   832  
   833  SECURITY WARNING
   834  This download is listed in go.sum, but using an unknown hash algorithm.
   835  The download cannot be verified.
   836  
   837  For more information, see 'go help module-auth'.
   838  
   839  `
   840  
   841  var HelpModuleAuth = &base.Command{
   842  	UsageLine: "module-auth",
   843  	Short:     "module authentication using go.sum",
   844  	Long: `
   845  When the go command downloads a module zip file or go.mod file into the
   846  module cache, it computes a cryptographic hash and compares it with a known
   847  value to verify the file hasn't changed since it was first downloaded. Known
   848  hashes are stored in a file in the module root directory named go.sum. Hashes
   849  may also be downloaded from the checksum database depending on the values of
   850  GOSUMDB, GOPRIVATE, and GONOSUMDB.
   851  
   852  For details, see https://golang.org/ref/mod#authenticating.
   853  `,
   854  }
   855  
   856  var HelpPrivate = &base.Command{
   857  	UsageLine: "private",
   858  	Short:     "configuration for downloading non-public code",
   859  	Long: `
   860  The go command defaults to downloading modules from the public Go module
   861  mirror at proxy.golang.org. It also defaults to validating downloaded modules,
   862  regardless of source, against the public Go checksum database at sum.golang.org.
   863  These defaults work well for publicly available source code.
   864  
   865  The GOPRIVATE environment variable controls which modules the go command
   866  considers to be private (not available publicly) and should therefore not use
   867  the proxy or checksum database. The variable is a comma-separated list of
   868  glob patterns (in the syntax of Go's path.Match) of module path prefixes.
   869  For example,
   870  
   871  	GOPRIVATE=*.corp.example.com,rsc.io/private
   872  
   873  causes the go command to treat as private any module with a path prefix
   874  matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private,
   875  and rsc.io/private/quux.
   876  
   877  For fine-grained control over module download and validation, the GONOPROXY
   878  and GONOSUMDB environment variables accept the same kind of glob list
   879  and override GOPRIVATE for the specific decision of whether to use the proxy
   880  and checksum database, respectively.
   881  
   882  For example, if a company ran a module proxy serving private modules,
   883  users would configure go using:
   884  
   885  	GOPRIVATE=*.corp.example.com
   886  	GOPROXY=proxy.example.com
   887  	GONOPROXY=none
   888  
   889  The GOPRIVATE variable is also used to define the "public" and "private"
   890  patterns for the GOVCS variable; see 'go help vcs'. For that usage,
   891  GOPRIVATE applies even in GOPATH mode. In that case, it matches import paths
   892  instead of module paths.
   893  
   894  The 'go env -w' command (see 'go help env') can be used to set these variables
   895  for future go command invocations.
   896  
   897  For more details, see https://golang.org/ref/mod#private-modules.
   898  `,
   899  }
   900  

View as plain text