Source file src/cmd/go/internal/modfetch/repo.go
Documentation: cmd/go/internal/modfetch
1 // Copyright 2018 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package modfetch 6 7 import ( 8 "fmt" 9 "io" 10 "io/fs" 11 "os" 12 "strconv" 13 "time" 14 15 "cmd/go/internal/cfg" 16 "cmd/go/internal/modfetch/codehost" 17 "cmd/go/internal/par" 18 "cmd/go/internal/vcs" 19 web "cmd/go/internal/web" 20 21 "golang.org/x/mod/module" 22 ) 23 24 const traceRepo = false // trace all repo actions, for debugging 25 26 // A Repo represents a repository storing all versions of a single module. 27 // It must be safe for simultaneous use by multiple goroutines. 28 type Repo interface { 29 // ModulePath returns the module path. 30 ModulePath() string 31 32 // Versions lists all known versions with the given prefix. 33 // Pseudo-versions are not included. 34 // 35 // Versions should be returned sorted in semver order 36 // (implementations can use semver.Sort). 37 // 38 // Versions returns a non-nil error only if there was a problem 39 // fetching the list of versions: it may return an empty list 40 // along with a nil error if the list of matching versions 41 // is known to be empty. 42 // 43 // If the underlying repository does not exist, 44 // Versions returns an error matching errors.Is(_, os.NotExist). 45 Versions(prefix string) ([]string, error) 46 47 // Stat returns information about the revision rev. 48 // A revision can be any identifier known to the underlying service: 49 // commit hash, branch, tag, and so on. 50 Stat(rev string) (*RevInfo, error) 51 52 // Latest returns the latest revision on the default branch, 53 // whatever that means in the underlying source code repository. 54 // It is only used when there are no tagged versions. 55 Latest() (*RevInfo, error) 56 57 // GoMod returns the go.mod file for the given version. 58 GoMod(version string) (data []byte, err error) 59 60 // Zip writes a zip file for the given version to dst. 61 Zip(dst io.Writer, version string) error 62 } 63 64 // A Rev describes a single revision in a module repository. 65 type RevInfo struct { 66 Version string // suggested version string for this revision 67 Time time.Time // commit time 68 69 // These fields are used for Stat of arbitrary rev, 70 // but they are not recorded when talking about module versions. 71 Name string `json:"-"` // complete ID in underlying repository 72 Short string `json:"-"` // shortened ID, for use in pseudo-version 73 } 74 75 // Re: module paths, import paths, repository roots, and lookups 76 // 77 // A module is a collection of Go packages stored in a file tree 78 // with a go.mod file at the root of the tree. 79 // The go.mod defines the module path, which is the import path 80 // corresponding to the root of the file tree. 81 // The import path of a directory within that file tree is the module path 82 // joined with the name of the subdirectory relative to the root. 83 // 84 // For example, the module with path rsc.io/qr corresponds to the 85 // file tree in the repository https://github.com/rsc/qr. 86 // That file tree has a go.mod that says "module rsc.io/qr". 87 // The package in the root directory has import path "rsc.io/qr". 88 // The package in the gf256 subdirectory has import path "rsc.io/qr/gf256". 89 // In this example, "rsc.io/qr" is both a module path and an import path. 90 // But "rsc.io/qr/gf256" is only an import path, not a module path: 91 // it names an importable package, but not a module. 92 // 93 // As a special case to incorporate code written before modules were 94 // introduced, if a path p resolves using the pre-module "go get" lookup 95 // to the root of a source code repository without a go.mod file, 96 // that repository is treated as if it had a go.mod in its root directory 97 // declaring module path p. (The go.mod is further considered to 98 // contain requirements corresponding to any legacy version 99 // tracking format such as Gopkg.lock, vendor/vendor.conf, and so on.) 100 // 101 // The presentation so far ignores the fact that a source code repository 102 // has many different versions of a file tree, and those versions may 103 // differ in whether a particular go.mod exists and what it contains. 104 // In fact there is a well-defined mapping only from a module path, version 105 // pair - often written path@version - to a particular file tree. 106 // For example rsc.io/qr@v0.1.0 depends on the "implicit go.mod at root of 107 // repository" rule, while rsc.io/qr@v0.2.0 has an explicit go.mod. 108 // Because the "go get" import paths rsc.io/qr and github.com/rsc/qr 109 // both redirect to the Git repository https://github.com/rsc/qr, 110 // github.com/rsc/qr@v0.1.0 is the same file tree as rsc.io/qr@v0.1.0 111 // but a different module (a different name). In contrast, since v0.2.0 112 // of that repository has an explicit go.mod that declares path rsc.io/qr, 113 // github.com/rsc/qr@v0.2.0 is an invalid module path, version pair. 114 // Before modules, import comments would have had the same effect. 115 // 116 // The set of import paths associated with a given module path is 117 // clearly not fixed: at the least, new directories with new import paths 118 // can always be added. But another potential operation is to split a 119 // subtree out of a module into its own module. If done carefully, 120 // this operation can be done while preserving compatibility for clients. 121 // For example, suppose that we want to split rsc.io/qr/gf256 into its 122 // own module, so that there would be two modules rsc.io/qr and rsc.io/qr/gf256. 123 // Then we can simultaneously issue rsc.io/qr v0.3.0 (dropping the gf256 subdirectory) 124 // and rsc.io/qr/gf256 v0.1.0, including in their respective go.mod 125 // cyclic requirements pointing at each other: rsc.io/qr v0.3.0 requires 126 // rsc.io/qr/gf256 v0.1.0 and vice versa. Then a build can be 127 // using an older rsc.io/qr module that includes the gf256 package, but if 128 // it adds a requirement on either the newer rsc.io/qr or the newer 129 // rsc.io/qr/gf256 module, it will automatically add the requirement 130 // on the complementary half, ensuring both that rsc.io/qr/gf256 is 131 // available for importing by the build and also that it is only defined 132 // by a single module. The gf256 package could move back into the 133 // original by another simultaneous release of rsc.io/qr v0.4.0 including 134 // the gf256 subdirectory and an rsc.io/qr/gf256 v0.2.0 with no code 135 // in its root directory, along with a new requirement cycle. 136 // The ability to shift module boundaries in this way is expected to be 137 // important in large-scale program refactorings, similar to the ones 138 // described in https://talks.golang.org/2016/refactor.article. 139 // 140 // The possibility of shifting module boundaries reemphasizes 141 // that you must know both the module path and its version 142 // to determine the set of packages provided directly by that module. 143 // 144 // On top of all this, it is possible for a single code repository 145 // to contain multiple modules, either in branches or subdirectories, 146 // as a limited kind of monorepo. For example rsc.io/qr/v2, 147 // the v2.x.x continuation of rsc.io/qr, is expected to be found 148 // in v2-tagged commits in https://github.com/rsc/qr, either 149 // in the root or in a v2 subdirectory, disambiguated by go.mod. 150 // Again the precise file tree corresponding to a module 151 // depends on which version we are considering. 152 // 153 // It is also possible for the underlying repository to change over time, 154 // without changing the module path. If I copy the github repo over 155 // to https://bitbucket.org/rsc/qr and update https://rsc.io/qr?go-get=1, 156 // then clients of all versions should start fetching from bitbucket 157 // instead of github. That is, in contrast to the exact file tree, 158 // the location of the source code repository associated with a module path 159 // does not depend on the module version. (This is by design, as the whole 160 // point of these redirects is to allow package authors to establish a stable 161 // name that can be updated as code moves from one service to another.) 162 // 163 // All of this is important background for the lookup APIs defined in this 164 // file. 165 // 166 // The Lookup function takes a module path and returns a Repo representing 167 // that module path. Lookup can do only a little with the path alone. 168 // It can check that the path is well-formed (see semver.CheckPath) 169 // and it can check that the path can be resolved to a target repository. 170 // To avoid version control access except when absolutely necessary, 171 // Lookup does not attempt to connect to the repository itself. 172 173 var lookupCache par.Cache 174 175 type lookupCacheKey struct { 176 proxy, path string 177 } 178 179 // Lookup returns the module with the given module path, 180 // fetched through the given proxy. 181 // 182 // The distinguished proxy "direct" indicates that the path should be fetched 183 // from its origin, and "noproxy" indicates that the patch should be fetched 184 // directly only if GONOPROXY matches the given path. 185 // 186 // For the distinguished proxy "off", Lookup always returns a Repo that returns 187 // a non-nil error for every method call. 188 // 189 // A successful return does not guarantee that the module 190 // has any defined versions. 191 func Lookup(proxy, path string) Repo { 192 if traceRepo { 193 defer logCall("Lookup(%q, %q)", proxy, path)() 194 } 195 196 type cached struct { 197 r Repo 198 } 199 c := lookupCache.Do(lookupCacheKey{proxy, path}, func() interface{} { 200 r := newCachingRepo(path, func() (Repo, error) { 201 r, err := lookup(proxy, path) 202 if err == nil && traceRepo { 203 r = newLoggingRepo(r) 204 } 205 return r, err 206 }) 207 return cached{r} 208 }).(cached) 209 210 return c.r 211 } 212 213 // lookup returns the module with the given module path. 214 func lookup(proxy, path string) (r Repo, err error) { 215 if cfg.BuildMod == "vendor" { 216 return nil, errLookupDisabled 217 } 218 219 if module.MatchPrefixPatterns(cfg.GONOPROXY, path) { 220 switch proxy { 221 case "noproxy", "direct": 222 return lookupDirect(path) 223 default: 224 return nil, errNoproxy 225 } 226 } 227 228 switch proxy { 229 case "off": 230 return errRepo{path, errProxyOff}, nil 231 case "direct": 232 return lookupDirect(path) 233 case "noproxy": 234 return nil, errUseProxy 235 default: 236 return newProxyRepo(proxy, path) 237 } 238 } 239 240 type lookupDisabledError struct{} 241 242 func (lookupDisabledError) Error() string { 243 if cfg.BuildModReason == "" { 244 return fmt.Sprintf("module lookup disabled by -mod=%s", cfg.BuildMod) 245 } 246 return fmt.Sprintf("module lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) 247 } 248 249 var errLookupDisabled error = lookupDisabledError{} 250 251 var ( 252 errProxyOff = notExistErrorf("module lookup disabled by GOPROXY=off") 253 errNoproxy error = notExistErrorf("disabled by GOPRIVATE/GONOPROXY") 254 errUseProxy error = notExistErrorf("path does not match GOPRIVATE/GONOPROXY") 255 ) 256 257 func lookupDirect(path string) (Repo, error) { 258 security := web.SecureOnly 259 260 if module.MatchPrefixPatterns(cfg.GOINSECURE, path) { 261 security = web.Insecure 262 } 263 rr, err := vcs.RepoRootForImportPath(path, vcs.PreferMod, security) 264 if err != nil { 265 // We don't know where to find code for a module with this path. 266 return nil, notExistError{err: err} 267 } 268 269 if rr.VCS.Name == "mod" { 270 // Fetch module from proxy with base URL rr.Repo. 271 return newProxyRepo(rr.Repo, path) 272 } 273 274 code, err := lookupCodeRepo(rr) 275 if err != nil { 276 return nil, err 277 } 278 return newCodeRepo(code, rr.Root, path) 279 } 280 281 func lookupCodeRepo(rr *vcs.RepoRoot) (codehost.Repo, error) { 282 code, err := codehost.NewRepo(rr.VCS.Cmd, rr.Repo) 283 if err != nil { 284 if _, ok := err.(*codehost.VCSError); ok { 285 return nil, err 286 } 287 return nil, fmt.Errorf("lookup %s: %v", rr.Root, err) 288 } 289 return code, nil 290 } 291 292 // A loggingRepo is a wrapper around an underlying Repo 293 // that prints a log message at the start and end of each call. 294 // It can be inserted when debugging. 295 type loggingRepo struct { 296 r Repo 297 } 298 299 func newLoggingRepo(r Repo) *loggingRepo { 300 return &loggingRepo{r} 301 } 302 303 // logCall prints a log message using format and args and then 304 // also returns a function that will print the same message again, 305 // along with the elapsed time. 306 // Typical usage is: 307 // 308 // defer logCall("hello %s", arg)() 309 // 310 // Note the final (). 311 func logCall(format string, args ...interface{}) func() { 312 start := time.Now() 313 fmt.Fprintf(os.Stderr, "+++ %s\n", fmt.Sprintf(format, args...)) 314 return func() { 315 fmt.Fprintf(os.Stderr, "%.3fs %s\n", time.Since(start).Seconds(), fmt.Sprintf(format, args...)) 316 } 317 } 318 319 func (l *loggingRepo) ModulePath() string { 320 return l.r.ModulePath() 321 } 322 323 func (l *loggingRepo) Versions(prefix string) (tags []string, err error) { 324 defer logCall("Repo[%s]: Versions(%q)", l.r.ModulePath(), prefix)() 325 return l.r.Versions(prefix) 326 } 327 328 func (l *loggingRepo) Stat(rev string) (*RevInfo, error) { 329 defer logCall("Repo[%s]: Stat(%q)", l.r.ModulePath(), rev)() 330 return l.r.Stat(rev) 331 } 332 333 func (l *loggingRepo) Latest() (*RevInfo, error) { 334 defer logCall("Repo[%s]: Latest()", l.r.ModulePath())() 335 return l.r.Latest() 336 } 337 338 func (l *loggingRepo) GoMod(version string) ([]byte, error) { 339 defer logCall("Repo[%s]: GoMod(%q)", l.r.ModulePath(), version)() 340 return l.r.GoMod(version) 341 } 342 343 func (l *loggingRepo) Zip(dst io.Writer, version string) error { 344 dstName := "_" 345 if dst, ok := dst.(interface{ Name() string }); ok { 346 dstName = strconv.Quote(dst.Name()) 347 } 348 defer logCall("Repo[%s]: Zip(%s, %q)", l.r.ModulePath(), dstName, version)() 349 return l.r.Zip(dst, version) 350 } 351 352 // errRepo is a Repo that returns the same error for all operations. 353 // 354 // It is useful in conjunction with caching, since cache hits will not attempt 355 // the prohibited operations. 356 type errRepo struct { 357 modulePath string 358 err error 359 } 360 361 func (r errRepo) ModulePath() string { return r.modulePath } 362 363 func (r errRepo) Versions(prefix string) (tags []string, err error) { return nil, r.err } 364 func (r errRepo) Stat(rev string) (*RevInfo, error) { return nil, r.err } 365 func (r errRepo) Latest() (*RevInfo, error) { return nil, r.err } 366 func (r errRepo) GoMod(version string) ([]byte, error) { return nil, r.err } 367 func (r errRepo) Zip(dst io.Writer, version string) error { return r.err } 368 369 // A notExistError is like fs.ErrNotExist, but with a custom message 370 type notExistError struct { 371 err error 372 } 373 374 func notExistErrorf(format string, args ...interface{}) error { 375 return notExistError{fmt.Errorf(format, args...)} 376 } 377 378 func (e notExistError) Error() string { 379 return e.err.Error() 380 } 381 382 func (notExistError) Is(target error) bool { 383 return target == fs.ErrNotExist 384 } 385 386 func (e notExistError) Unwrap() error { 387 return e.err 388 } 389