proxy.go

Documentation: cmd/go/internal/modfetch

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modfetch
     6  
     7  import (
     8  	"encoding/json"
     9  	"errors"
    10  	"fmt"
    11  	"io"
    12  	"io/fs"
    13  	"net/url"
    14  	"path"
    15  	pathpkg "path"
    16  	"path/filepath"
    17  	"strings"
    18  	"sync"
    19  	"time"
    20  
    21  	"cmd/go/internal/base"
    22  	"cmd/go/internal/cfg"
    23  	"cmd/go/internal/modfetch/codehost"
    24  	"cmd/go/internal/web"
    25  
    26  	"golang.org/x/mod/module"
    27  	"golang.org/x/mod/semver"
    28  )
    29  
    30  var HelpGoproxy = &base.Command{
    31  	UsageLine: "goproxy",
    32  	Short:     "module proxy protocol",
    33  	Long: `
    34  A Go module proxy is any web server that can respond to GET requests for
    35  URLs of a specified form. The requests have no query parameters, so even
    36  a site serving from a fixed file system (including a file:/// URL)
    37  can be a module proxy.
    38  
    39  For details on the GOPROXY protocol, see
    40  https://golang.org/ref/mod#goproxy-protocol.
    41  `,
    42  }
    43  
    44  var proxyOnce struct {
    45  	sync.Once
    46  	list []proxySpec
    47  	err  error
    48  }
    49  
    50  type proxySpec struct {
    51  	// url is the proxy URL or one of "off", "direct", "noproxy".
    52  	url string
    53  
    54  	// fallBackOnError is true if a request should be attempted on the next proxy
    55  	// in the list after any error from this proxy. If fallBackOnError is false,
    56  	// the request will only be attempted on the next proxy if the error is
    57  	// equivalent to os.ErrNotFound, which is true for 404 and 410 responses.
    58  	fallBackOnError bool
    59  }
    60  
    61  func proxyList() ([]proxySpec, error) {
    62  	proxyOnce.Do(func() {
    63  		if cfg.GONOPROXY != "" && cfg.GOPROXY != "direct" {
    64  			proxyOnce.list = append(proxyOnce.list, proxySpec{url: "noproxy"})
    65  		}
    66  
    67  		goproxy := cfg.GOPROXY
    68  		for goproxy != "" {
    69  			var url string
    70  			fallBackOnError := false
    71  			if i := strings.IndexAny(goproxy, ",|"); i >= 0 {
    72  				url = goproxy[:i]
    73  				fallBackOnError = goproxy[i] == '|'
    74  				goproxy = goproxy[i+1:]
    75  			} else {
    76  				url = goproxy
    77  				goproxy = ""
    78  			}
    79  
    80  			url = strings.TrimSpace(url)
    81  			if url == "" {
    82  				continue
    83  			}
    84  			if url == "off" {
    85  				// "off" always fails hard, so can stop walking list.
    86  				proxyOnce.list = append(proxyOnce.list, proxySpec{url: "off"})
    87  				break
    88  			}
    89  			if url == "direct" {
    90  				proxyOnce.list = append(proxyOnce.list, proxySpec{url: "direct"})
    91  				// For now, "direct" is the end of the line. We may decide to add some
    92  				// sort of fallback behavior for them in the future, so ignore
    93  				// subsequent entries for forward-compatibility.
    94  				break
    95  			}
    96  
    97  			// Single-word tokens are reserved for built-in behaviors, and anything
    98  			// containing the string ":/" or matching an absolute file path must be a
    99  			// complete URL. For all other paths, implicitly add "https://".
   100  			if strings.ContainsAny(url, ".:/") && !strings.Contains(url, ":/") && !filepath.IsAbs(url) && !path.IsAbs(url) {
   101  				url = "https://" + url
   102  			}
   103  
   104  			// Check that newProxyRepo accepts the URL.
   105  			// It won't do anything with the path.
   106  			if _, err := newProxyRepo(url, "golang.org/x/text"); err != nil {
   107  				proxyOnce.err = err
   108  				return
   109  			}
   110  
   111  			proxyOnce.list = append(proxyOnce.list, proxySpec{
   112  				url:             url,
   113  				fallBackOnError: fallBackOnError,
   114  			})
   115  		}
   116  
   117  		if len(proxyOnce.list) == 0 ||
   118  			len(proxyOnce.list) == 1 && proxyOnce.list[0].url == "noproxy" {
   119  			// There were no proxies, other than the implicit "noproxy" added when
   120  			// GONOPROXY is set. This can happen if GOPROXY is a non-empty string
   121  			// like "," or " ".
   122  			proxyOnce.err = fmt.Errorf("GOPROXY list is not the empty string, but contains no entries")
   123  		}
   124  	})
   125  
   126  	return proxyOnce.list, proxyOnce.err
   127  }
   128  
   129  // TryProxies iterates f over each configured proxy (including "noproxy" and
   130  // "direct" if applicable) until f returns no error or until f returns an
   131  // error that is not equivalent to fs.ErrNotExist on a proxy configured
   132  // not to fall back on errors.
   133  //
   134  // TryProxies then returns that final error.
   135  //
   136  // If GOPROXY is set to "off", TryProxies invokes f once with the argument
   137  // "off".
   138  func TryProxies(f func(proxy string) error) error {
   139  	proxies, err := proxyList()
   140  	if err != nil {
   141  		return err
   142  	}
   143  	if len(proxies) == 0 {
   144  		panic("GOPROXY list is empty")
   145  	}
   146  
   147  	// We try to report the most helpful error to the user. "direct" and "noproxy"
   148  	// errors are best, followed by proxy errors other than ErrNotExist, followed
   149  	// by ErrNotExist.
   150  	//
   151  	// Note that errProxyOff, errNoproxy, and errUseProxy are equivalent to
   152  	// ErrNotExist. errUseProxy should only be returned if "noproxy" is the only
   153  	// proxy. errNoproxy should never be returned, since there should always be a
   154  	// more useful error from "noproxy" first.
   155  	const (
   156  		notExistRank = iota
   157  		proxyRank
   158  		directRank
   159  	)
   160  	var bestErr error
   161  	bestErrRank := notExistRank
   162  	for _, proxy := range proxies {
   163  		err := f(proxy.url)
   164  		if err == nil {
   165  			return nil
   166  		}
   167  		isNotExistErr := errors.Is(err, fs.ErrNotExist)
   168  
   169  		if proxy.url == "direct" || (proxy.url == "noproxy" && err != errUseProxy) {
   170  			bestErr = err
   171  			bestErrRank = directRank
   172  		} else if bestErrRank <= proxyRank && !isNotExistErr {
   173  			bestErr = err
   174  			bestErrRank = proxyRank
   175  		} else if bestErrRank == notExistRank {
   176  			bestErr = err
   177  		}
   178  
   179  		if !proxy.fallBackOnError && !isNotExistErr {
   180  			break
   181  		}
   182  	}
   183  	return bestErr
   184  }
   185  
   186  type proxyRepo struct {
   187  	url         *url.URL
   188  	path        string
   189  	redactedURL string
   190  }
   191  
   192  func newProxyRepo(baseURL, path string) (Repo, error) {
   193  	base, err := url.Parse(baseURL)
   194  	if err != nil {
   195  		return nil, err
   196  	}
   197  	switch base.Scheme {
   198  	case "http", "https":
   199  		// ok
   200  	case "file":
   201  		if *base != (url.URL{Scheme: base.Scheme, Path: base.Path, RawPath: base.RawPath}) {
   202  			return nil, fmt.Errorf("invalid file:// proxy URL with non-path elements: %s", base.Redacted())
   203  		}
   204  	case "":
   205  		return nil, fmt.Errorf("invalid proxy URL missing scheme: %s", base.Redacted())
   206  	default:
   207  		return nil, fmt.Errorf("invalid proxy URL scheme (must be https, http, file): %s", base.Redacted())
   208  	}
   209  
   210  	enc, err := module.EscapePath(path)
   211  	if err != nil {
   212  		return nil, err
   213  	}
   214  	redactedURL := base.Redacted()
   215  	base.Path = strings.TrimSuffix(base.Path, "/") + "/" + enc
   216  	base.RawPath = strings.TrimSuffix(base.RawPath, "/") + "/" + pathEscape(enc)
   217  	return &proxyRepo{base, path, redactedURL}, nil
   218  }
   219  
   220  func (p *proxyRepo) ModulePath() string {
   221  	return p.path
   222  }
   223  
   224  // versionError returns err wrapped in a ModuleError for p.path.
   225  func (p *proxyRepo) versionError(version string, err error) error {
   226  	if version != "" && version != module.CanonicalVersion(version) {
   227  		return &module.ModuleError{
   228  			Path: p.path,
   229  			Err: &module.InvalidVersionError{
   230  				Version: version,
   231  				Pseudo:  module.IsPseudoVersion(version),
   232  				Err:     err,
   233  			},
   234  		}
   235  	}
   236  
   237  	return &module.ModuleError{
   238  		Path:    p.path,
   239  		Version: version,
   240  		Err:     err,
   241  	}
   242  }
   243  
   244  func (p *proxyRepo) getBytes(path string) ([]byte, error) {
   245  	body, err := p.getBody(path)
   246  	if err != nil {
   247  		return nil, err
   248  	}
   249  	defer body.Close()
   250  	return io.ReadAll(body)
   251  }
   252  
   253  func (p *proxyRepo) getBody(path string) (io.ReadCloser, error) {
   254  	fullPath := pathpkg.Join(p.url.Path, path)
   255  
   256  	target := *p.url
   257  	target.Path = fullPath
   258  	target.RawPath = pathpkg.Join(target.RawPath, pathEscape(path))
   259  
   260  	resp, err := web.Get(web.DefaultSecurity, &target)
   261  	if err != nil {
   262  		return nil, err
   263  	}
   264  	if err := resp.Err(); err != nil {
   265  		resp.Body.Close()
   266  		return nil, err
   267  	}
   268  	return resp.Body, nil
   269  }
   270  
   271  func (p *proxyRepo) Versions(prefix string) ([]string, error) {
   272  	data, err := p.getBytes("@v/list")
   273  	if err != nil {
   274  		return nil, p.versionError("", err)
   275  	}
   276  	var list []string
   277  	for _, line := range strings.Split(string(data), "\n") {
   278  		f := strings.Fields(line)
   279  		if len(f) >= 1 && semver.IsValid(f[0]) && strings.HasPrefix(f[0], prefix) && !module.IsPseudoVersion(f[0]) {
   280  			list = append(list, f[0])
   281  		}
   282  	}
   283  	semver.Sort(list)
   284  	return list, nil
   285  }
   286  
   287  func (p *proxyRepo) latest() (*RevInfo, error) {
   288  	data, err := p.getBytes("@v/list")
   289  	if err != nil {
   290  		return nil, p.versionError("", err)
   291  	}
   292  
   293  	var (
   294  		bestTime             time.Time
   295  		bestTimeIsFromPseudo bool
   296  		bestVersion          string
   297  	)
   298  
   299  	for _, line := range strings.Split(string(data), "\n") {
   300  		f := strings.Fields(line)
   301  		if len(f) >= 1 && semver.IsValid(f[0]) {
   302  			// If the proxy includes timestamps, prefer the timestamp it reports.
   303  			// Otherwise, derive the timestamp from the pseudo-version.
   304  			var (
   305  				ft             time.Time
   306  				ftIsFromPseudo = false
   307  			)
   308  			if len(f) >= 2 {
   309  				ft, _ = time.Parse(time.RFC3339, f[1])
   310  			} else if module.IsPseudoVersion(f[0]) {
   311  				ft, _ = module.PseudoVersionTime(f[0])
   312  				ftIsFromPseudo = true
   313  			} else {
   314  				// Repo.Latest promises that this method is only called where there are
   315  				// no tagged versions. Ignore any tagged versions that were added in the
   316  				// meantime.
   317  				continue
   318  			}
   319  			if bestTime.Before(ft) {
   320  				bestTime = ft
   321  				bestTimeIsFromPseudo = ftIsFromPseudo
   322  				bestVersion = f[0]
   323  			}
   324  		}
   325  	}
   326  	if bestVersion == "" {
   327  		return nil, p.versionError("", codehost.ErrNoCommits)
   328  	}
   329  
   330  	if bestTimeIsFromPseudo {
   331  		// We parsed bestTime from the pseudo-version, but that's in UTC and we're
   332  		// supposed to report the timestamp as reported by the VCS.
   333  		// Stat the selected version to canonicalize the timestamp.
   334  		//
   335  		// TODO(bcmills): Should we also stat other versions to ensure that we
   336  		// report the correct Name and Short for the revision?
   337  		return p.Stat(bestVersion)
   338  	}
   339  
   340  	return &RevInfo{
   341  		Version: bestVersion,
   342  		Name:    bestVersion,
   343  		Short:   bestVersion,
   344  		Time:    bestTime,
   345  	}, nil
   346  }
   347  
   348  func (p *proxyRepo) Stat(rev string) (*RevInfo, error) {
   349  	encRev, err := module.EscapeVersion(rev)
   350  	if err != nil {
   351  		return nil, p.versionError(rev, err)
   352  	}
   353  	data, err := p.getBytes("@v/" + encRev + ".info")
   354  	if err != nil {
   355  		return nil, p.versionError(rev, err)
   356  	}
   357  	info := new(RevInfo)
   358  	if err := json.Unmarshal(data, info); err != nil {
   359  		return nil, p.versionError(rev, fmt.Errorf("invalid response from proxy %q: %w", p.redactedURL, err))
   360  	}
   361  	if info.Version != rev && rev == module.CanonicalVersion(rev) && module.Check(p.path, rev) == nil {
   362  		// If we request a correct, appropriate version for the module path, the
   363  		// proxy must return either exactly that version or an error — not some
   364  		// arbitrary other version.
   365  		return nil, p.versionError(rev, fmt.Errorf("proxy returned info for version %s instead of requested version", info.Version))
   366  	}
   367  	return info, nil
   368  }
   369  
   370  func (p *proxyRepo) Latest() (*RevInfo, error) {
   371  	data, err := p.getBytes("@latest")
   372  	if err != nil {
   373  		if !errors.Is(err, fs.ErrNotExist) {
   374  			return nil, p.versionError("", err)
   375  		}
   376  		return p.latest()
   377  	}
   378  	info := new(RevInfo)
   379  	if err := json.Unmarshal(data, info); err != nil {
   380  		return nil, p.versionError("", fmt.Errorf("invalid response from proxy %q: %w", p.redactedURL, err))
   381  	}
   382  	return info, nil
   383  }
   384  
   385  func (p *proxyRepo) GoMod(version string) ([]byte, error) {
   386  	if version != module.CanonicalVersion(version) {
   387  		return nil, p.versionError(version, fmt.Errorf("internal error: version passed to GoMod is not canonical"))
   388  	}
   389  
   390  	encVer, err := module.EscapeVersion(version)
   391  	if err != nil {
   392  		return nil, p.versionError(version, err)
   393  	}
   394  	data, err := p.getBytes("@v/" + encVer + ".mod")
   395  	if err != nil {
   396  		return nil, p.versionError(version, err)
   397  	}
   398  	return data, nil
   399  }
   400  
   401  func (p *proxyRepo) Zip(dst io.Writer, version string) error {
   402  	if version != module.CanonicalVersion(version) {
   403  		return p.versionError(version, fmt.Errorf("internal error: version passed to Zip is not canonical"))
   404  	}
   405  
   406  	encVer, err := module.EscapeVersion(version)
   407  	if err != nil {
   408  		return p.versionError(version, err)
   409  	}
   410  	body, err := p.getBody("@v/" + encVer + ".zip")
   411  	if err != nil {
   412  		return p.versionError(version, err)
   413  	}
   414  	defer body.Close()
   415  
   416  	lr := &io.LimitedReader{R: body, N: codehost.MaxZipFile + 1}
   417  	if _, err := io.Copy(dst, lr); err != nil {
   418  		return p.versionError(version, err)
   419  	}
   420  	if lr.N <= 0 {
   421  		return p.versionError(version, fmt.Errorf("downloaded zip file too large"))
   422  	}
   423  	return nil
   424  }
   425  
   426  // pathEscape escapes s so it can be used in a path.
   427  // That is, it escapes things like ? and # (which really shouldn't appear anyway).
   428  // It does not escape / to %2F: our REST API is designed so that / can be left as is.
   429  func pathEscape(s string) string {
   430  	return strings.ReplaceAll(url.PathEscape(s), "%2F", "/")
   431  }
   432
View as plain text