sumdb.go

Documentation: cmd/go/internal/modfetch

     1  // Copyright 2019 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Go checksum database lookup
     6  
     7  //go:build !cmd_go_bootstrap
     8  // +build !cmd_go_bootstrap
     9  
    10  package modfetch
    11  
    12  import (
    13  	"bytes"
    14  	"errors"
    15  	"fmt"
    16  	"io"
    17  	"io/fs"
    18  	"net/url"
    19  	"os"
    20  	"path/filepath"
    21  	"strings"
    22  	"sync"
    23  	"time"
    24  
    25  	"cmd/go/internal/base"
    26  	"cmd/go/internal/cfg"
    27  	"cmd/go/internal/lockedfile"
    28  	"cmd/go/internal/web"
    29  
    30  	"golang.org/x/mod/module"
    31  	"golang.org/x/mod/sumdb"
    32  	"golang.org/x/mod/sumdb/note"
    33  )
    34  
    35  // useSumDB reports whether to use the Go checksum database for the given module.
    36  func useSumDB(mod module.Version) bool {
    37  	return cfg.GOSUMDB != "off" && !module.MatchPrefixPatterns(cfg.GONOSUMDB, mod.Path)
    38  }
    39  
    40  // lookupSumDB returns the Go checksum database's go.sum lines for the given module,
    41  // along with the name of the database.
    42  func lookupSumDB(mod module.Version) (dbname string, lines []string, err error) {
    43  	dbOnce.Do(func() {
    44  		dbName, db, dbErr = dbDial()
    45  	})
    46  	if dbErr != nil {
    47  		return "", nil, dbErr
    48  	}
    49  	lines, err = db.Lookup(mod.Path, mod.Version)
    50  	return dbName, lines, err
    51  }
    52  
    53  var (
    54  	dbOnce sync.Once
    55  	dbName string
    56  	db     *sumdb.Client
    57  	dbErr  error
    58  )
    59  
    60  func dbDial() (dbName string, db *sumdb.Client, err error) {
    61  	// $GOSUMDB can be "key" or "key url",
    62  	// and the key can be a full verifier key
    63  	// or a host on our list of known keys.
    64  
    65  	// Special case: sum.golang.google.cn
    66  	// is an alias, reachable inside mainland China,
    67  	// for sum.golang.org. If there are more
    68  	// of these we should add a map like knownGOSUMDB.
    69  	gosumdb := cfg.GOSUMDB
    70  	if gosumdb == "sum.golang.google.cn" {
    71  		gosumdb = "sum.golang.org https://sum.golang.google.cn"
    72  	}
    73  
    74  	key := strings.Fields(gosumdb)
    75  	if len(key) >= 1 {
    76  		if k := knownGOSUMDB[key[0]]; k != "" {
    77  			key[0] = k
    78  		}
    79  	}
    80  	if len(key) == 0 {
    81  		return "", nil, fmt.Errorf("missing GOSUMDB")
    82  	}
    83  	if len(key) > 2 {
    84  		return "", nil, fmt.Errorf("invalid GOSUMDB: too many fields")
    85  	}
    86  	vkey, err := note.NewVerifier(key[0])
    87  	if err != nil {
    88  		return "", nil, fmt.Errorf("invalid GOSUMDB: %v", err)
    89  	}
    90  	name := vkey.Name()
    91  
    92  	// No funny business in the database name.
    93  	direct, err := url.Parse("https://" + name)
    94  	if err != nil || strings.HasSuffix(name, "/") || *direct != (url.URL{Scheme: "https", Host: direct.Host, Path: direct.Path, RawPath: direct.RawPath}) || direct.RawPath != "" || direct.Host == "" {
    95  		return "", nil, fmt.Errorf("invalid sumdb name (must be host[/path]): %s %+v", name, *direct)
    96  	}
    97  
    98  	// Determine how to get to database.
    99  	var base *url.URL
   100  	if len(key) >= 2 {
   101  		// Use explicit alternate URL listed in $GOSUMDB,
   102  		// bypassing both the default URL derivation and any proxies.
   103  		u, err := url.Parse(key[1])
   104  		if err != nil {
   105  			return "", nil, fmt.Errorf("invalid GOSUMDB URL: %v", err)
   106  		}
   107  		base = u
   108  	}
   109  
   110  	return name, sumdb.NewClient(&dbClient{key: key[0], name: name, direct: direct, base: base}), nil
   111  }
   112  
   113  type dbClient struct {
   114  	key    string
   115  	name   string
   116  	direct *url.URL
   117  
   118  	once    sync.Once
   119  	base    *url.URL
   120  	baseErr error
   121  }
   122  
   123  func (c *dbClient) ReadRemote(path string) ([]byte, error) {
   124  	c.once.Do(c.initBase)
   125  	if c.baseErr != nil {
   126  		return nil, c.baseErr
   127  	}
   128  
   129  	var data []byte
   130  	start := time.Now()
   131  	targ := web.Join(c.base, path)
   132  	data, err := web.GetBytes(targ)
   133  	if false {
   134  		fmt.Fprintf(os.Stderr, "%.3fs %s\n", time.Since(start).Seconds(), targ.Redacted())
   135  	}
   136  	return data, err
   137  }
   138  
   139  // initBase determines the base URL for connecting to the database.
   140  // Determining the URL requires sending network traffic to proxies,
   141  // so this work is delayed until we need to download something from
   142  // the database. If everything we need is in the local cache and
   143  // c.ReadRemote is never called, we will never do this work.
   144  func (c *dbClient) initBase() {
   145  	if c.base != nil {
   146  		return
   147  	}
   148  
   149  	// Try proxies in turn until we find out how to connect to this database.
   150  	//
   151  	// Before accessing any checksum database URL using a proxy, the proxy
   152  	// client should first fetch <proxyURL>/sumdb/<sumdb-name>/supported.
   153  	//
   154  	// If that request returns a successful (HTTP 200) response, then the proxy
   155  	// supports proxying checksum database requests. In that case, the client
   156  	// should use the proxied access method only, never falling back to a direct
   157  	// connection to the database.
   158  	//
   159  	// If the /sumdb/<sumdb-name>/supported check fails with a “not found” (HTTP
   160  	// 404) or “gone” (HTTP 410) response, or if the proxy is configured to fall
   161  	// back on errors, the client will try the next proxy. If there are no
   162  	// proxies left or if the proxy is "direct" or "off", the client should
   163  	// connect directly to that database.
   164  	//
   165  	// Any other response is treated as the database being unavailable.
   166  	//
   167  	// See https://golang.org/design/25530-sumdb#proxying-a-checksum-database.
   168  	err := TryProxies(func(proxy string) error {
   169  		switch proxy {
   170  		case "noproxy":
   171  			return errUseProxy
   172  		case "direct", "off":
   173  			return errProxyOff
   174  		default:
   175  			proxyURL, err := url.Parse(proxy)
   176  			if err != nil {
   177  				return err
   178  			}
   179  			if _, err := web.GetBytes(web.Join(proxyURL, "sumdb/"+c.name+"/supported")); err != nil {
   180  				return err
   181  			}
   182  			// Success! This proxy will help us.
   183  			c.base = web.Join(proxyURL, "sumdb/"+c.name)
   184  			return nil
   185  		}
   186  	})
   187  	if errors.Is(err, fs.ErrNotExist) {
   188  		// No proxies, or all proxies failed (with 404, 410, or were allowed
   189  		// to fall back), or we reached an explicit "direct" or "off".
   190  		c.base = c.direct
   191  	} else if err != nil {
   192  		c.baseErr = err
   193  	}
   194  }
   195  
   196  // ReadConfig reads the key from c.key
   197  // and otherwise reads the config (a latest tree head) from GOPATH/pkg/sumdb/<file>.
   198  func (c *dbClient) ReadConfig(file string) (data []byte, err error) {
   199  	if file == "key" {
   200  		return []byte(c.key), nil
   201  	}
   202  
   203  	if cfg.SumdbDir == "" {
   204  		return nil, errors.New("could not locate sumdb file: missing $GOPATH")
   205  	}
   206  	targ := filepath.Join(cfg.SumdbDir, file)
   207  	data, err = lockedfile.Read(targ)
   208  	if errors.Is(err, fs.ErrNotExist) {
   209  		// Treat non-existent as empty, to bootstrap the "latest" file
   210  		// the first time we connect to a given database.
   211  		return []byte{}, nil
   212  	}
   213  	return data, err
   214  }
   215  
   216  // WriteConfig rewrites the latest tree head.
   217  func (*dbClient) WriteConfig(file string, old, new []byte) error {
   218  	if file == "key" {
   219  		// Should not happen.
   220  		return fmt.Errorf("cannot write key")
   221  	}
   222  	if cfg.SumdbDir == "" {
   223  		return errors.New("could not locate sumdb file: missing $GOPATH")
   224  	}
   225  	targ := filepath.Join(cfg.SumdbDir, file)
   226  	os.MkdirAll(filepath.Dir(targ), 0777)
   227  	f, err := lockedfile.Edit(targ)
   228  	if err != nil {
   229  		return err
   230  	}
   231  	defer f.Close()
   232  	data, err := io.ReadAll(f)
   233  	if err != nil {
   234  		return err
   235  	}
   236  	if len(data) > 0 && !bytes.Equal(data, old) {
   237  		return sumdb.ErrWriteConflict
   238  	}
   239  	if _, err := f.Seek(0, 0); err != nil {
   240  		return err
   241  	}
   242  	if err := f.Truncate(0); err != nil {
   243  		return err
   244  	}
   245  	if _, err := f.Write(new); err != nil {
   246  		return err
   247  	}
   248  	return f.Close()
   249  }
   250  
   251  // ReadCache reads cached lookups or tiles from
   252  // GOPATH/pkg/mod/cache/download/sumdb,
   253  // which will be deleted by "go clean -modcache".
   254  func (*dbClient) ReadCache(file string) ([]byte, error) {
   255  	targ := filepath.Join(cfg.GOMODCACHE, "cache/download/sumdb", file)
   256  	data, err := lockedfile.Read(targ)
   257  	// lockedfile.Write does not atomically create the file with contents.
   258  	// There is a moment between file creation and locking the file for writing,
   259  	// during which the empty file can be locked for reading.
   260  	// Treat observing an empty file as file not found.
   261  	if err == nil && len(data) == 0 {
   262  		err = &fs.PathError{Op: "read", Path: targ, Err: fs.ErrNotExist}
   263  	}
   264  	return data, err
   265  }
   266  
   267  // WriteCache updates cached lookups or tiles.
   268  func (*dbClient) WriteCache(file string, data []byte) {
   269  	targ := filepath.Join(cfg.GOMODCACHE, "cache/download/sumdb", file)
   270  	os.MkdirAll(filepath.Dir(targ), 0777)
   271  	lockedfile.Write(targ, bytes.NewReader(data), 0666)
   272  }
   273  
   274  func (*dbClient) Log(msg string) {
   275  	// nothing for now
   276  }
   277  
   278  func (*dbClient) SecurityError(msg string) {
   279  	base.Fatalf("%s", msg)
   280  }
   281
View as plain text