1
2
3
4
5 package vcs
6
7 import (
8 "encoding/json"
9 "errors"
10 "fmt"
11 exec "internal/execabs"
12 "internal/lazyregexp"
13 "internal/singleflight"
14 "io/fs"
15 "log"
16 urlpkg "net/url"
17 "os"
18 "path/filepath"
19 "regexp"
20 "strings"
21 "sync"
22
23 "cmd/go/internal/base"
24 "cmd/go/internal/cfg"
25 "cmd/go/internal/search"
26 "cmd/go/internal/str"
27 "cmd/go/internal/web"
28
29 "golang.org/x/mod/module"
30 )
31
32
33
34 type Cmd struct {
35 Name string
36 Cmd string
37
38 CreateCmd []string
39 DownloadCmd []string
40
41 TagCmd []tagCmd
42 TagLookupCmd []tagCmd
43 TagSyncCmd []string
44 TagSyncDefault []string
45
46 Scheme []string
47 PingCmd string
48
49 RemoteRepo func(v *Cmd, rootDir string) (remoteRepo string, err error)
50 ResolveRepo func(v *Cmd, rootDir, remoteRepo string) (realRepo string, err error)
51 }
52
53 var defaultSecureScheme = map[string]bool{
54 "https": true,
55 "git+ssh": true,
56 "bzr+ssh": true,
57 "svn+ssh": true,
58 "ssh": true,
59 }
60
61 func (v *Cmd) IsSecure(repo string) bool {
62 u, err := urlpkg.Parse(repo)
63 if err != nil {
64
65 return false
66 }
67 return v.isSecureScheme(u.Scheme)
68 }
69
70 func (v *Cmd) isSecureScheme(scheme string) bool {
71 switch v.Cmd {
72 case "git":
73
74
75
76 if allow := os.Getenv("GIT_ALLOW_PROTOCOL"); allow != "" {
77 for _, s := range strings.Split(allow, ":") {
78 if s == scheme {
79 return true
80 }
81 }
82 return false
83 }
84 }
85 return defaultSecureScheme[scheme]
86 }
87
88
89
90 type tagCmd struct {
91 cmd string
92 pattern string
93 }
94
95
96 var vcsList = []*Cmd{
97 vcsHg,
98 vcsGit,
99 vcsSvn,
100 vcsBzr,
101 vcsFossil,
102 }
103
104
105
106 var vcsMod = &Cmd{Name: "mod"}
107
108
109
110 func vcsByCmd(cmd string) *Cmd {
111 for _, vcs := range vcsList {
112 if vcs.Cmd == cmd {
113 return vcs
114 }
115 }
116 return nil
117 }
118
119
120 var vcsHg = &Cmd{
121 Name: "Mercurial",
122 Cmd: "hg",
123
124 CreateCmd: []string{"clone -U -- {repo} {dir}"},
125 DownloadCmd: []string{"pull"},
126
127
128
129
130
131
132 TagCmd: []tagCmd{
133 {"tags", `^(\S+)`},
134 {"branches", `^(\S+)`},
135 },
136 TagSyncCmd: []string{"update -r {tag}"},
137 TagSyncDefault: []string{"update default"},
138
139 Scheme: []string{"https", "http", "ssh"},
140 PingCmd: "identify -- {scheme}://{repo}",
141 RemoteRepo: hgRemoteRepo,
142 }
143
144 func hgRemoteRepo(vcsHg *Cmd, rootDir string) (remoteRepo string, err error) {
145 out, err := vcsHg.runOutput(rootDir, "paths default")
146 if err != nil {
147 return "", err
148 }
149 return strings.TrimSpace(string(out)), nil
150 }
151
152
153 var vcsGit = &Cmd{
154 Name: "Git",
155 Cmd: "git",
156
157 CreateCmd: []string{"clone -- {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"},
158 DownloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"},
159
160 TagCmd: []tagCmd{
161
162
163 {"show-ref", `(?:tags|origin)/(\S+)$`},
164 },
165 TagLookupCmd: []tagCmd{
166 {"show-ref tags/{tag} origin/{tag}", `((?:tags|origin)/\S+)$`},
167 },
168 TagSyncCmd: []string{"checkout {tag}", "submodule update --init --recursive"},
169
170
171
172
173
174 TagSyncDefault: []string{"submodule update --init --recursive"},
175
176 Scheme: []string{"git", "https", "http", "git+ssh", "ssh"},
177
178
179
180
181
182 PingCmd: "ls-remote {scheme}://{repo}",
183
184 RemoteRepo: gitRemoteRepo,
185 }
186
187
188
189 var scpSyntaxRe = lazyregexp.New(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`)
190
191 func gitRemoteRepo(vcsGit *Cmd, rootDir string) (remoteRepo string, err error) {
192 cmd := "config remote.origin.url"
193 errParse := errors.New("unable to parse output of git " + cmd)
194 errRemoteOriginNotFound := errors.New("remote origin not found")
195 outb, err := vcsGit.run1(rootDir, cmd, nil, false)
196 if err != nil {
197
198
199 if outb != nil && len(outb) == 0 {
200 return "", errRemoteOriginNotFound
201 }
202 return "", err
203 }
204 out := strings.TrimSpace(string(outb))
205
206 var repoURL *urlpkg.URL
207 if m := scpSyntaxRe.FindStringSubmatch(out); m != nil {
208
209
210
211 repoURL = &urlpkg.URL{
212 Scheme: "ssh",
213 User: urlpkg.User(m[1]),
214 Host: m[2],
215 Path: m[3],
216 }
217 } else {
218 repoURL, err = urlpkg.Parse(out)
219 if err != nil {
220 return "", err
221 }
222 }
223
224
225
226
227 for _, s := range vcsGit.Scheme {
228 if repoURL.Scheme == s {
229 return repoURL.String(), nil
230 }
231 }
232 return "", errParse
233 }
234
235
236 var vcsBzr = &Cmd{
237 Name: "Bazaar",
238 Cmd: "bzr",
239
240 CreateCmd: []string{"branch -- {repo} {dir}"},
241
242
243
244 DownloadCmd: []string{"pull --overwrite"},
245
246 TagCmd: []tagCmd{{"tags", `^(\S+)`}},
247 TagSyncCmd: []string{"update -r {tag}"},
248 TagSyncDefault: []string{"update -r revno:-1"},
249
250 Scheme: []string{"https", "http", "bzr", "bzr+ssh"},
251 PingCmd: "info -- {scheme}://{repo}",
252 RemoteRepo: bzrRemoteRepo,
253 ResolveRepo: bzrResolveRepo,
254 }
255
256 func bzrRemoteRepo(vcsBzr *Cmd, rootDir string) (remoteRepo string, err error) {
257 outb, err := vcsBzr.runOutput(rootDir, "config parent_location")
258 if err != nil {
259 return "", err
260 }
261 return strings.TrimSpace(string(outb)), nil
262 }
263
264 func bzrResolveRepo(vcsBzr *Cmd, rootDir, remoteRepo string) (realRepo string, err error) {
265 outb, err := vcsBzr.runOutput(rootDir, "info "+remoteRepo)
266 if err != nil {
267 return "", err
268 }
269 out := string(outb)
270
271
272
273
274
275
276 found := false
277 for _, prefix := range []string{"\n branch root: ", "\n repository branch: "} {
278 i := strings.Index(out, prefix)
279 if i >= 0 {
280 out = out[i+len(prefix):]
281 found = true
282 break
283 }
284 }
285 if !found {
286 return "", fmt.Errorf("unable to parse output of bzr info")
287 }
288
289 i := strings.Index(out, "\n")
290 if i < 0 {
291 return "", fmt.Errorf("unable to parse output of bzr info")
292 }
293 out = out[:i]
294 return strings.TrimSpace(out), nil
295 }
296
297
298 var vcsSvn = &Cmd{
299 Name: "Subversion",
300 Cmd: "svn",
301
302 CreateCmd: []string{"checkout -- {repo} {dir}"},
303 DownloadCmd: []string{"update"},
304
305
306
307
308 Scheme: []string{"https", "http", "svn", "svn+ssh"},
309 PingCmd: "info -- {scheme}://{repo}",
310 RemoteRepo: svnRemoteRepo,
311 }
312
313 func svnRemoteRepo(vcsSvn *Cmd, rootDir string) (remoteRepo string, err error) {
314 outb, err := vcsSvn.runOutput(rootDir, "info")
315 if err != nil {
316 return "", err
317 }
318 out := string(outb)
319
320
321
322
323
324
325
326
327
328
329
330 i := strings.Index(out, "\nURL: ")
331 if i < 0 {
332 return "", fmt.Errorf("unable to parse output of svn info")
333 }
334 out = out[i+len("\nURL: "):]
335 i = strings.Index(out, "\n")
336 if i < 0 {
337 return "", fmt.Errorf("unable to parse output of svn info")
338 }
339 out = out[:i]
340 return strings.TrimSpace(out), nil
341 }
342
343
344
345 const fossilRepoName = ".fossil"
346
347
348 var vcsFossil = &Cmd{
349 Name: "Fossil",
350 Cmd: "fossil",
351
352 CreateCmd: []string{"-go-internal-mkdir {dir} clone -- {repo} " + filepath.Join("{dir}", fossilRepoName), "-go-internal-cd {dir} open .fossil"},
353 DownloadCmd: []string{"up"},
354
355 TagCmd: []tagCmd{{"tag ls", `(.*)`}},
356 TagSyncCmd: []string{"up tag:{tag}"},
357 TagSyncDefault: []string{"up trunk"},
358
359 Scheme: []string{"https", "http"},
360 RemoteRepo: fossilRemoteRepo,
361 }
362
363 func fossilRemoteRepo(vcsFossil *Cmd, rootDir string) (remoteRepo string, err error) {
364 out, err := vcsFossil.runOutput(rootDir, "remote-url")
365 if err != nil {
366 return "", err
367 }
368 return strings.TrimSpace(string(out)), nil
369 }
370
371 func (v *Cmd) String() string {
372 return v.Name
373 }
374
375
376
377
378
379
380
381
382 func (v *Cmd) run(dir string, cmd string, keyval ...string) error {
383 _, err := v.run1(dir, cmd, keyval, true)
384 return err
385 }
386
387
388 func (v *Cmd) runVerboseOnly(dir string, cmd string, keyval ...string) error {
389 _, err := v.run1(dir, cmd, keyval, false)
390 return err
391 }
392
393
394 func (v *Cmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) {
395 return v.run1(dir, cmd, keyval, true)
396 }
397
398
399 func (v *Cmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) {
400 m := make(map[string]string)
401 for i := 0; i < len(keyval); i += 2 {
402 m[keyval[i]] = keyval[i+1]
403 }
404 args := strings.Fields(cmdline)
405 for i, arg := range args {
406 args[i] = expand(m, arg)
407 }
408
409 if len(args) >= 2 && args[0] == "-go-internal-mkdir" {
410 var err error
411 if filepath.IsAbs(args[1]) {
412 err = os.Mkdir(args[1], fs.ModePerm)
413 } else {
414 err = os.Mkdir(filepath.Join(dir, args[1]), fs.ModePerm)
415 }
416 if err != nil {
417 return nil, err
418 }
419 args = args[2:]
420 }
421
422 if len(args) >= 2 && args[0] == "-go-internal-cd" {
423 if filepath.IsAbs(args[1]) {
424 dir = args[1]
425 } else {
426 dir = filepath.Join(dir, args[1])
427 }
428 args = args[2:]
429 }
430
431 _, err := exec.LookPath(v.Cmd)
432 if err != nil {
433 fmt.Fprintf(os.Stderr,
434 "go: missing %s command. See https://golang.org/s/gogetcmd\n",
435 v.Name)
436 return nil, err
437 }
438
439 cmd := exec.Command(v.Cmd, args...)
440 cmd.Dir = dir
441 cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir)
442 if cfg.BuildX {
443 fmt.Fprintf(os.Stderr, "cd %s\n", dir)
444 fmt.Fprintf(os.Stderr, "%s %s\n", v.Cmd, strings.Join(args, " "))
445 }
446 out, err := cmd.Output()
447 if err != nil {
448 if verbose || cfg.BuildV {
449 fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.Cmd, strings.Join(args, " "))
450 if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 {
451 os.Stderr.Write(ee.Stderr)
452 } else {
453 fmt.Fprintf(os.Stderr, err.Error())
454 }
455 }
456 }
457 return out, err
458 }
459
460
461 func (v *Cmd) Ping(scheme, repo string) error {
462 return v.runVerboseOnly(".", v.PingCmd, "scheme", scheme, "repo", repo)
463 }
464
465
466
467 func (v *Cmd) Create(dir, repo string) error {
468 for _, cmd := range v.CreateCmd {
469 if err := v.run(".", cmd, "dir", dir, "repo", repo); err != nil {
470 return err
471 }
472 }
473 return nil
474 }
475
476
477 func (v *Cmd) Download(dir string) error {
478 for _, cmd := range v.DownloadCmd {
479 if err := v.run(dir, cmd); err != nil {
480 return err
481 }
482 }
483 return nil
484 }
485
486
487 func (v *Cmd) Tags(dir string) ([]string, error) {
488 var tags []string
489 for _, tc := range v.TagCmd {
490 out, err := v.runOutput(dir, tc.cmd)
491 if err != nil {
492 return nil, err
493 }
494 re := regexp.MustCompile(`(?m-s)` + tc.pattern)
495 for _, m := range re.FindAllStringSubmatch(string(out), -1) {
496 tags = append(tags, m[1])
497 }
498 }
499 return tags, nil
500 }
501
502
503
504 func (v *Cmd) TagSync(dir, tag string) error {
505 if v.TagSyncCmd == nil {
506 return nil
507 }
508 if tag != "" {
509 for _, tc := range v.TagLookupCmd {
510 out, err := v.runOutput(dir, tc.cmd, "tag", tag)
511 if err != nil {
512 return err
513 }
514 re := regexp.MustCompile(`(?m-s)` + tc.pattern)
515 m := re.FindStringSubmatch(string(out))
516 if len(m) > 1 {
517 tag = m[1]
518 break
519 }
520 }
521 }
522
523 if tag == "" && v.TagSyncDefault != nil {
524 for _, cmd := range v.TagSyncDefault {
525 if err := v.run(dir, cmd); err != nil {
526 return err
527 }
528 }
529 return nil
530 }
531
532 for _, cmd := range v.TagSyncCmd {
533 if err := v.run(dir, cmd, "tag", tag); err != nil {
534 return err
535 }
536 }
537 return nil
538 }
539
540
541
542 type vcsPath struct {
543 pathPrefix string
544 regexp *lazyregexp.Regexp
545 repo string
546 vcs string
547 check func(match map[string]string) error
548 schemelessRepo bool
549 }
550
551
552
553
554
555 func FromDir(dir, srcRoot string) (vcs *Cmd, root string, err error) {
556
557 dir = filepath.Clean(dir)
558 srcRoot = filepath.Clean(srcRoot)
559 if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator {
560 return nil, "", fmt.Errorf("directory %q is outside source root %q", dir, srcRoot)
561 }
562
563 var vcsRet *Cmd
564 var rootRet string
565
566 origDir := dir
567 for len(dir) > len(srcRoot) {
568 for _, vcs := range vcsList {
569 if _, err := os.Stat(filepath.Join(dir, "."+vcs.Cmd)); err == nil {
570 root := filepath.ToSlash(dir[len(srcRoot)+1:])
571
572
573 if vcsRet == nil {
574 vcsRet = vcs
575 rootRet = root
576 continue
577 }
578
579 if vcsRet == vcs && vcs.Cmd == "git" {
580 continue
581 }
582
583 return nil, "", fmt.Errorf("directory %q uses %s, but parent %q uses %s",
584 filepath.Join(srcRoot, rootRet), vcsRet.Cmd, filepath.Join(srcRoot, root), vcs.Cmd)
585 }
586 }
587
588
589 ndir := filepath.Dir(dir)
590 if len(ndir) >= len(dir) {
591
592 break
593 }
594 dir = ndir
595 }
596
597 if vcsRet != nil {
598 if err := checkGOVCS(vcsRet, rootRet); err != nil {
599 return nil, "", err
600 }
601 return vcsRet, rootRet, nil
602 }
603
604 return nil, "", fmt.Errorf("directory %q is not using a known version control system", origDir)
605 }
606
607
608 type govcsRule struct {
609 pattern string
610 allowed []string
611 }
612
613
614 type govcsConfig []govcsRule
615
616 func parseGOVCS(s string) (govcsConfig, error) {
617 s = strings.TrimSpace(s)
618 if s == "" {
619 return nil, nil
620 }
621 var cfg govcsConfig
622 have := make(map[string]string)
623 for _, item := range strings.Split(s, ",") {
624 item = strings.TrimSpace(item)
625 if item == "" {
626 return nil, fmt.Errorf("empty entry in GOVCS")
627 }
628 i := strings.Index(item, ":")
629 if i < 0 {
630 return nil, fmt.Errorf("malformed entry in GOVCS (missing colon): %q", item)
631 }
632 pattern, list := strings.TrimSpace(item[:i]), strings.TrimSpace(item[i+1:])
633 if pattern == "" {
634 return nil, fmt.Errorf("empty pattern in GOVCS: %q", item)
635 }
636 if list == "" {
637 return nil, fmt.Errorf("empty VCS list in GOVCS: %q", item)
638 }
639 if search.IsRelativePath(pattern) {
640 return nil, fmt.Errorf("relative pattern not allowed in GOVCS: %q", pattern)
641 }
642 if old := have[pattern]; old != "" {
643 return nil, fmt.Errorf("unreachable pattern in GOVCS: %q after %q", item, old)
644 }
645 have[pattern] = item
646 allowed := strings.Split(list, "|")
647 for i, a := range allowed {
648 a = strings.TrimSpace(a)
649 if a == "" {
650 return nil, fmt.Errorf("empty VCS name in GOVCS: %q", item)
651 }
652 allowed[i] = a
653 }
654 cfg = append(cfg, govcsRule{pattern, allowed})
655 }
656 return cfg, nil
657 }
658
659 func (c *govcsConfig) allow(path string, private bool, vcs string) bool {
660 for _, rule := range *c {
661 match := false
662 switch rule.pattern {
663 case "private":
664 match = private
665 case "public":
666 match = !private
667 default:
668
669
670 match = module.MatchPrefixPatterns(rule.pattern, path)
671 }
672 if !match {
673 continue
674 }
675 for _, allow := range rule.allowed {
676 if allow == vcs || allow == "all" {
677 return true
678 }
679 }
680 return false
681 }
682
683
684 return false
685 }
686
687 var (
688 govcs govcsConfig
689 govcsErr error
690 govcsOnce sync.Once
691 )
692
693
694
695
696
697
698
699
700
701
702
703
704
705 var defaultGOVCS = govcsConfig{
706 {"private", []string{"all"}},
707 {"public", []string{"git", "hg"}},
708 }
709
710 func checkGOVCS(vcs *Cmd, root string) error {
711 if vcs == vcsMod {
712
713
714
715 return nil
716 }
717
718 govcsOnce.Do(func() {
719 govcs, govcsErr = parseGOVCS(os.Getenv("GOVCS"))
720 govcs = append(govcs, defaultGOVCS...)
721 })
722 if govcsErr != nil {
723 return govcsErr
724 }
725
726 private := module.MatchPrefixPatterns(cfg.GOPRIVATE, root)
727 if !govcs.allow(root, private, vcs.Cmd) {
728 what := "public"
729 if private {
730 what = "private"
731 }
732 return fmt.Errorf("GOVCS disallows using %s for %s %s; see 'go help vcs'", vcs.Cmd, what, root)
733 }
734
735 return nil
736 }
737
738
739
740 func CheckNested(vcs *Cmd, dir, srcRoot string) error {
741 if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator {
742 return fmt.Errorf("directory %q is outside source root %q", dir, srcRoot)
743 }
744
745 otherDir := dir
746 for len(otherDir) > len(srcRoot) {
747 for _, otherVCS := range vcsList {
748 if _, err := os.Stat(filepath.Join(otherDir, "."+otherVCS.Cmd)); err == nil {
749
750 if otherDir == dir && otherVCS == vcs {
751 continue
752 }
753
754 if otherVCS == vcs && vcs.Cmd == "git" {
755 continue
756 }
757
758 return fmt.Errorf("directory %q uses %s, but parent %q uses %s", dir, vcs.Cmd, otherDir, otherVCS.Cmd)
759 }
760 }
761
762 newDir := filepath.Dir(otherDir)
763 if len(newDir) >= len(otherDir) {
764
765 break
766 }
767 otherDir = newDir
768 }
769
770 return nil
771 }
772
773
774 type RepoRoot struct {
775 Repo string
776 Root string
777 IsCustom bool
778 VCS *Cmd
779 }
780
781 func httpPrefix(s string) string {
782 for _, prefix := range [...]string{"http:", "https:"} {
783 if strings.HasPrefix(s, prefix) {
784 return prefix
785 }
786 }
787 return ""
788 }
789
790
791 type ModuleMode int
792
793 const (
794 IgnoreMod ModuleMode = iota
795 PreferMod
796 )
797
798
799
800 func RepoRootForImportPath(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) {
801 rr, err := repoRootFromVCSPaths(importPath, security, vcsPaths)
802 if err == errUnknownSite {
803 rr, err = repoRootForImportDynamic(importPath, mod, security)
804 if err != nil {
805 err = importErrorf(importPath, "unrecognized import path %q: %v", importPath, err)
806 }
807 }
808 if err != nil {
809 rr1, err1 := repoRootFromVCSPaths(importPath, security, vcsPathsAfterDynamic)
810 if err1 == nil {
811 rr = rr1
812 err = nil
813 }
814 }
815
816
817 if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.Root, "...") {
818
819 rr = nil
820 err = importErrorf(importPath, "cannot expand ... in %q", importPath)
821 }
822 return rr, err
823 }
824
825 var errUnknownSite = errors.New("dynamic lookup required to find mapping")
826
827
828
829 func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths []*vcsPath) (*RepoRoot, error) {
830 if str.HasPathPrefix(importPath, "example.net") {
831
832
833
834
835 return nil, fmt.Errorf("no modules on example.net")
836 }
837 if importPath == "rsc.io" {
838
839
840
841
842 return nil, fmt.Errorf("rsc.io is not a module")
843 }
844
845
846 if prefix := httpPrefix(importPath); prefix != "" {
847
848
849 return nil, fmt.Errorf("%q not allowed in import path", prefix+"//")
850 }
851 for _, srv := range vcsPaths {
852 if !str.HasPathPrefix(importPath, srv.pathPrefix) {
853 continue
854 }
855 m := srv.regexp.FindStringSubmatch(importPath)
856 if m == nil {
857 if srv.pathPrefix != "" {
858 return nil, importErrorf(importPath, "invalid %s import path %q", srv.pathPrefix, importPath)
859 }
860 continue
861 }
862
863
864 match := map[string]string{
865 "prefix": srv.pathPrefix + "/",
866 "import": importPath,
867 }
868 for i, name := range srv.regexp.SubexpNames() {
869 if name != "" && match[name] == "" {
870 match[name] = m[i]
871 }
872 }
873 if srv.vcs != "" {
874 match["vcs"] = expand(match, srv.vcs)
875 }
876 if srv.repo != "" {
877 match["repo"] = expand(match, srv.repo)
878 }
879 if srv.check != nil {
880 if err := srv.check(match); err != nil {
881 return nil, err
882 }
883 }
884 vcs := vcsByCmd(match["vcs"])
885 if vcs == nil {
886 return nil, fmt.Errorf("unknown version control system %q", match["vcs"])
887 }
888 if err := checkGOVCS(vcs, match["root"]); err != nil {
889 return nil, err
890 }
891 var repoURL string
892 if !srv.schemelessRepo {
893 repoURL = match["repo"]
894 } else {
895 scheme := vcs.Scheme[0]
896 repo := match["repo"]
897 if vcs.PingCmd != "" {
898
899 for _, s := range vcs.Scheme {
900 if security == web.SecureOnly && !vcs.isSecureScheme(s) {
901 continue
902 }
903 if vcs.Ping(s, repo) == nil {
904 scheme = s
905 break
906 }
907 }
908 }
909 repoURL = scheme + "://" + repo
910 }
911 rr := &RepoRoot{
912 Repo: repoURL,
913 Root: match["root"],
914 VCS: vcs,
915 }
916 return rr, nil
917 }
918 return nil, errUnknownSite
919 }
920
921
922
923
924
925 func urlForImportPath(importPath string) (*urlpkg.URL, error) {
926 slash := strings.Index(importPath, "/")
927 if slash < 0 {
928 slash = len(importPath)
929 }
930 host, path := importPath[:slash], importPath[slash:]
931 if !strings.Contains(host, ".") {
932 return nil, errors.New("import path does not begin with hostname")
933 }
934 if len(path) == 0 {
935 path = "/"
936 }
937 return &urlpkg.URL{Host: host, Path: path, RawQuery: "go-get=1"}, nil
938 }
939
940
941
942
943
944 func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) {
945 url, err := urlForImportPath(importPath)
946 if err != nil {
947 return nil, err
948 }
949 resp, err := web.Get(security, url)
950 if err != nil {
951 msg := "https fetch: %v"
952 if security == web.Insecure {
953 msg = "http/" + msg
954 }
955 return nil, fmt.Errorf(msg, err)
956 }
957 body := resp.Body
958 defer body.Close()
959 imports, err := parseMetaGoImports(body, mod)
960 if len(imports) == 0 {
961 if respErr := resp.Err(); respErr != nil {
962
963
964 return nil, respErr
965 }
966 }
967 if err != nil {
968 return nil, fmt.Errorf("parsing %s: %v", importPath, err)
969 }
970
971 mmi, err := matchGoImport(imports, importPath)
972 if err != nil {
973 if _, ok := err.(ImportMismatchError); !ok {
974 return nil, fmt.Errorf("parse %s: %v", url, err)
975 }
976 return nil, fmt.Errorf("parse %s: no go-import meta tags (%s)", resp.URL, err)
977 }
978 if cfg.BuildV {
979 log.Printf("get %q: found meta tag %#v at %s", importPath, mmi, url)
980 }
981
982
983
984
985
986
987 if mmi.Prefix != importPath {
988 if cfg.BuildV {
989 log.Printf("get %q: verifying non-authoritative meta tag", importPath)
990 }
991 var imports []metaImport
992 url, imports, err = metaImportsForPrefix(mmi.Prefix, mod, security)
993 if err != nil {
994 return nil, err
995 }
996 metaImport2, err := matchGoImport(imports, importPath)
997 if err != nil || mmi != metaImport2 {
998 return nil, fmt.Errorf("%s and %s disagree about go-import for %s", resp.URL, url, mmi.Prefix)
999 }
1000 }
1001
1002 if err := validateRepoRoot(mmi.RepoRoot); err != nil {
1003 return nil, fmt.Errorf("%s: invalid repo root %q: %v", resp.URL, mmi.RepoRoot, err)
1004 }
1005 var vcs *Cmd
1006 if mmi.VCS == "mod" {
1007 vcs = vcsMod
1008 } else {
1009 vcs = vcsByCmd(mmi.VCS)
1010 if vcs == nil {
1011 return nil, fmt.Errorf("%s: unknown vcs %q", resp.URL, mmi.VCS)
1012 }
1013 }
1014
1015 if err := checkGOVCS(vcs, mmi.Prefix); err != nil {
1016 return nil, err
1017 }
1018
1019 rr := &RepoRoot{
1020 Repo: mmi.RepoRoot,
1021 Root: mmi.Prefix,
1022 IsCustom: true,
1023 VCS: vcs,
1024 }
1025 return rr, nil
1026 }
1027
1028
1029
1030 func validateRepoRoot(repoRoot string) error {
1031 url, err := urlpkg.Parse(repoRoot)
1032 if err != nil {
1033 return err
1034 }
1035 if url.Scheme == "" {
1036 return errors.New("no scheme")
1037 }
1038 if url.Scheme == "file" {
1039 return errors.New("file scheme disallowed")
1040 }
1041 return nil
1042 }
1043
1044 var fetchGroup singleflight.Group
1045 var (
1046 fetchCacheMu sync.Mutex
1047 fetchCache = map[string]fetchResult{}
1048 )
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058 func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.SecurityMode) (*urlpkg.URL, []metaImport, error) {
1059 setCache := func(res fetchResult) (fetchResult, error) {
1060 fetchCacheMu.Lock()
1061 defer fetchCacheMu.Unlock()
1062 fetchCache[importPrefix] = res
1063 return res, nil
1064 }
1065
1066 resi, _, _ := fetchGroup.Do(importPrefix, func() (resi interface{}, err error) {
1067 fetchCacheMu.Lock()
1068 if res, ok := fetchCache[importPrefix]; ok {
1069 fetchCacheMu.Unlock()
1070 return res, nil
1071 }
1072 fetchCacheMu.Unlock()
1073
1074 url, err := urlForImportPath(importPrefix)
1075 if err != nil {
1076 return setCache(fetchResult{err: err})
1077 }
1078 resp, err := web.Get(security, url)
1079 if err != nil {
1080 return setCache(fetchResult{url: url, err: fmt.Errorf("fetching %s: %v", importPrefix, err)})
1081 }
1082 body := resp.Body
1083 defer body.Close()
1084 imports, err := parseMetaGoImports(body, mod)
1085 if len(imports) == 0 {
1086 if respErr := resp.Err(); respErr != nil {
1087
1088
1089 return setCache(fetchResult{url: url, err: respErr})
1090 }
1091 }
1092 if err != nil {
1093 return setCache(fetchResult{url: url, err: fmt.Errorf("parsing %s: %v", resp.URL, err)})
1094 }
1095 if len(imports) == 0 {
1096 err = fmt.Errorf("fetching %s: no go-import meta tag found in %s", importPrefix, resp.URL)
1097 }
1098 return setCache(fetchResult{url: url, imports: imports, err: err})
1099 })
1100 res := resi.(fetchResult)
1101 return res.url, res.imports, res.err
1102 }
1103
1104 type fetchResult struct {
1105 url *urlpkg.URL
1106 imports []metaImport
1107 err error
1108 }
1109
1110
1111
1112 type metaImport struct {
1113 Prefix, VCS, RepoRoot string
1114 }
1115
1116
1117
1118 type ImportMismatchError struct {
1119 importPath string
1120 mismatches []string
1121 }
1122
1123 func (m ImportMismatchError) Error() string {
1124 formattedStrings := make([]string, len(m.mismatches))
1125 for i, pre := range m.mismatches {
1126 formattedStrings[i] = fmt.Sprintf("meta tag %s did not match import path %s", pre, m.importPath)
1127 }
1128 return strings.Join(formattedStrings, ", ")
1129 }
1130
1131
1132
1133
1134 func matchGoImport(imports []metaImport, importPath string) (metaImport, error) {
1135 match := -1
1136
1137 errImportMismatch := ImportMismatchError{importPath: importPath}
1138 for i, im := range imports {
1139 if !str.HasPathPrefix(importPath, im.Prefix) {
1140 errImportMismatch.mismatches = append(errImportMismatch.mismatches, im.Prefix)
1141 continue
1142 }
1143
1144 if match >= 0 {
1145 if imports[match].VCS == "mod" && im.VCS != "mod" {
1146
1147
1148
1149 break
1150 }
1151 return metaImport{}, fmt.Errorf("multiple meta tags match import path %q", importPath)
1152 }
1153 match = i
1154 }
1155
1156 if match == -1 {
1157 return metaImport{}, errImportMismatch
1158 }
1159 return imports[match], nil
1160 }
1161
1162
1163 func expand(match map[string]string, s string) string {
1164
1165
1166
1167 oldNew := make([]string, 0, 2*len(match))
1168 for k, v := range match {
1169 oldNew = append(oldNew, "{"+k+"}", v)
1170 }
1171 return strings.NewReplacer(oldNew...).Replace(s)
1172 }
1173
1174
1175
1176
1177
1178 var vcsPaths = []*vcsPath{
1179
1180 {
1181 pathPrefix: "github.com",
1182 regexp: lazyregexp.New(`^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`),
1183 vcs: "git",
1184 repo: "https://{root}",
1185 check: noVCSSuffix,
1186 },
1187
1188
1189 {
1190 pathPrefix: "bitbucket.org",
1191 regexp: lazyregexp.New(`^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`),
1192 repo: "https://{root}",
1193 check: bitbucketVCS,
1194 },
1195
1196
1197 {
1198 pathPrefix: "hub.jazz.net/git",
1199 regexp: lazyregexp.New(`^(?P<root>hub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`),
1200 vcs: "git",
1201 repo: "https://{root}",
1202 check: noVCSSuffix,
1203 },
1204
1205
1206 {
1207 pathPrefix: "git.apache.org",
1208 regexp: lazyregexp.New(`^(?P<root>git\.apache\.org/[a-z0-9_.\-]+\.git)(/[A-Za-z0-9_.\-]+)*$`),
1209 vcs: "git",
1210 repo: "https://{root}",
1211 },
1212
1213
1214 {
1215 pathPrefix: "git.openstack.org",
1216 regexp: lazyregexp.New(`^(?P<root>git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/[A-Za-z0-9_.\-]+)*$`),
1217 vcs: "git",
1218 repo: "https://{root}",
1219 },
1220
1221
1222 {
1223 pathPrefix: "chiselapp.com",
1224 regexp: lazyregexp.New(`^(?P<root>chiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$`),
1225 vcs: "fossil",
1226 repo: "https://{root}",
1227 },
1228
1229
1230
1231 {
1232 regexp: lazyregexp.New(`(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\-]+)+?)\.(?P<vcs>bzr|fossil|git|hg|svn))(/~?[A-Za-z0-9_.\-]+)*$`),
1233 schemelessRepo: true,
1234 },
1235 }
1236
1237
1238
1239
1240
1241 var vcsPathsAfterDynamic = []*vcsPath{
1242
1243 {
1244 pathPrefix: "launchpad.net",
1245 regexp: lazyregexp.New(`^(?P<root>launchpad\.net/((?P<project>[A-Za-z0-9_.\-]+)(?P<series>/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`),
1246 vcs: "bzr",
1247 repo: "https://{root}",
1248 check: launchpadVCS,
1249 },
1250 }
1251
1252
1253
1254
1255 func noVCSSuffix(match map[string]string) error {
1256 repo := match["repo"]
1257 for _, vcs := range vcsList {
1258 if strings.HasSuffix(repo, "."+vcs.Cmd) {
1259 return fmt.Errorf("invalid version control suffix in %s path", match["prefix"])
1260 }
1261 }
1262 return nil
1263 }
1264
1265
1266
1267 func bitbucketVCS(match map[string]string) error {
1268 if err := noVCSSuffix(match); err != nil {
1269 return err
1270 }
1271
1272 var resp struct {
1273 SCM string `json:"scm"`
1274 }
1275 url := &urlpkg.URL{
1276 Scheme: "https",
1277 Host: "api.bitbucket.org",
1278 Path: expand(match, "/2.0/repositories/{bitname}"),
1279 RawQuery: "fields=scm",
1280 }
1281 data, err := web.GetBytes(url)
1282 if err != nil {
1283 if httpErr, ok := err.(*web.HTTPError); ok && httpErr.StatusCode == 403 {
1284
1285
1286 root := match["root"]
1287 for _, vcs := range []string{"git", "hg"} {
1288 if vcsByCmd(vcs).Ping("https", root) == nil {
1289 resp.SCM = vcs
1290 break
1291 }
1292 }
1293 }
1294
1295 if resp.SCM == "" {
1296 return err
1297 }
1298 } else {
1299 if err := json.Unmarshal(data, &resp); err != nil {
1300 return fmt.Errorf("decoding %s: %v", url, err)
1301 }
1302 }
1303
1304 if vcsByCmd(resp.SCM) != nil {
1305 match["vcs"] = resp.SCM
1306 if resp.SCM == "git" {
1307 match["repo"] += ".git"
1308 }
1309 return nil
1310 }
1311
1312 return fmt.Errorf("unable to detect version control system for bitbucket.org/ path")
1313 }
1314
1315
1316
1317
1318
1319 func launchpadVCS(match map[string]string) error {
1320 if match["project"] == "" || match["series"] == "" {
1321 return nil
1322 }
1323 url := &urlpkg.URL{
1324 Scheme: "https",
1325 Host: "code.launchpad.net",
1326 Path: expand(match, "/{project}{series}/.bzr/branch-format"),
1327 }
1328 _, err := web.GetBytes(url)
1329 if err != nil {
1330 match["root"] = expand(match, "launchpad.net/{project}")
1331 match["repo"] = expand(match, "https://{root}")
1332 }
1333 return nil
1334 }
1335
1336
1337
1338 type importError struct {
1339 importPath string
1340 err error
1341 }
1342
1343 func importErrorf(path, format string, args ...interface{}) error {
1344 err := &importError{importPath: path, err: fmt.Errorf(format, args...)}
1345 if errStr := err.Error(); !strings.Contains(errStr, path) {
1346 panic(fmt.Sprintf("path %q not in error %q", path, errStr))
1347 }
1348 return err
1349 }
1350
1351 func (e *importError) Error() string {
1352 return e.err.Error()
1353 }
1354
1355 func (e *importError) Unwrap() error {
1356
1357
1358 return errors.Unwrap(e.err)
1359 }
1360
1361 func (e *importError) ImportPath() string {
1362 return e.importPath
1363 }
1364
View as plain text