buildtool.go

Documentation: cmd/dist

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Build toolchain using Go 1.4.
     6  //
     7  // The general strategy is to copy the source files we need into
     8  // a new GOPATH workspace, adjust import paths appropriately,
     9  // invoke the Go 1.4 go command to build those sources,
    10  // and then copy the binaries back.
    11  
    12  package main
    13  
    14  import (
    15  	"fmt"
    16  	"os"
    17  	"path/filepath"
    18  	"runtime"
    19  	"strings"
    20  )
    21  
    22  // bootstrapDirs is a list of directories holding code that must be
    23  // compiled with a Go 1.4 toolchain to produce the bootstrapTargets.
    24  // All directories in this list are relative to and must be below $GOROOT/src.
    25  //
    26  // The list has two kinds of entries: names beginning with cmd/ with
    27  // no other slashes, which are commands, and other paths, which are packages
    28  // supporting the commands. Packages in the standard library can be listed
    29  // if a newer copy needs to be substituted for the Go 1.4 copy when used
    30  // by the command packages. Paths ending with /... automatically
    31  // include all packages within subdirectories as well.
    32  // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
    33  var bootstrapDirs = []string{
    34  	"cmd/asm",
    35  	"cmd/asm/internal/...",
    36  	"cmd/cgo",
    37  	"cmd/compile",
    38  	"cmd/compile/internal/...",
    39  	"cmd/internal/archive",
    40  	"cmd/internal/bio",
    41  	"cmd/internal/codesign",
    42  	"cmd/internal/dwarf",
    43  	"cmd/internal/edit",
    44  	"cmd/internal/gcprog",
    45  	"cmd/internal/goobj",
    46  	"cmd/internal/obj/...",
    47  	"cmd/internal/objabi",
    48  	"cmd/internal/pkgpath",
    49  	"cmd/internal/src",
    50  	"cmd/internal/sys",
    51  	"cmd/link",
    52  	"cmd/link/internal/...",
    53  	"compress/flate",
    54  	"compress/zlib",
    55  	"container/heap",
    56  	"debug/dwarf",
    57  	"debug/elf",
    58  	"debug/macho",
    59  	"debug/pe",
    60  	"go/constant",
    61  	"internal/buildcfg",
    62  	"internal/goexperiment",
    63  	"internal/goversion",
    64  	"internal/race",
    65  	"internal/unsafeheader",
    66  	"internal/xcoff",
    67  	"math/big",
    68  	"math/bits",
    69  	"sort",
    70  	"strconv",
    71  }
    72  
    73  // File prefixes that are ignored by go/build anyway, and cause
    74  // problems with editor generated temporary files (#18931).
    75  var ignorePrefixes = []string{
    76  	".",
    77  	"_",
    78  	"#",
    79  }
    80  
    81  // File suffixes that use build tags introduced since Go 1.4.
    82  // These must not be copied into the bootstrap build directory.
    83  // Also ignore test files.
    84  var ignoreSuffixes = []string{
    85  	"_arm64.s",
    86  	"_arm64.go",
    87  	"_riscv64.s",
    88  	"_riscv64.go",
    89  	"_wasm.s",
    90  	"_wasm.go",
    91  	"_test.s",
    92  	"_test.go",
    93  }
    94  
    95  func bootstrapBuildTools() {
    96  	goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
    97  	if goroot_bootstrap == "" {
    98  		goroot_bootstrap = pathf("%s/go1.4", os.Getenv("HOME"))
    99  	}
   100  	xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
   101  
   102  	mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
   103  	mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
   104  
   105  	// Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
   106  	// We use a subdirectory of $GOROOT/pkg because that's the
   107  	// space within $GOROOT where we store all generated objects.
   108  	// We could use a temporary directory outside $GOROOT instead,
   109  	// but it is easier to debug on failure if the files are in a known location.
   110  	workspace := pathf("%s/pkg/bootstrap", goroot)
   111  	xremoveall(workspace)
   112  	xatexit(func() { xremoveall(workspace) })
   113  	base := pathf("%s/src/bootstrap", workspace)
   114  	xmkdirall(base)
   115  
   116  	// Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
   117  	writefile("module bootstrap\n", pathf("%s/%s", base, "go.mod"), 0)
   118  	for _, dir := range bootstrapDirs {
   119  		recurse := strings.HasSuffix(dir, "/...")
   120  		dir = strings.TrimSuffix(dir, "/...")
   121  		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
   122  			if err != nil {
   123  				fatalf("walking bootstrap dirs failed: %v: %v", path, err)
   124  			}
   125  
   126  			name := filepath.Base(path)
   127  			src := pathf("%s/src/%s", goroot, path)
   128  			dst := pathf("%s/%s", base, path)
   129  
   130  			if info.IsDir() {
   131  				if !recurse && path != dir || name == "testdata" {
   132  					return filepath.SkipDir
   133  				}
   134  
   135  				xmkdirall(dst)
   136  				if path == "cmd/cgo" {
   137  					// Write to src because we need the file both for bootstrap
   138  					// and for later in the main build.
   139  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
   140  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
   141  				}
   142  				return nil
   143  			}
   144  
   145  			for _, pre := range ignorePrefixes {
   146  				if strings.HasPrefix(name, pre) {
   147  					return nil
   148  				}
   149  			}
   150  			for _, suf := range ignoreSuffixes {
   151  				if strings.HasSuffix(name, suf) {
   152  					return nil
   153  				}
   154  			}
   155  
   156  			text := bootstrapRewriteFile(src)
   157  			writefile(text, dst, 0)
   158  			return nil
   159  		})
   160  	}
   161  
   162  	// Set up environment for invoking Go 1.4 go command.
   163  	// GOROOT points at Go 1.4 GOROOT,
   164  	// GOPATH points at our bootstrap workspace,
   165  	// GOBIN is empty, so that binaries are installed to GOPATH/bin,
   166  	// and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
   167  	// so that Go 1.4 builds whatever kind of binary it knows how to build.
   168  	// Restore GOROOT, GOPATH, and GOBIN when done.
   169  	// Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
   170  	// because setup will take care of those when bootstrapBuildTools returns.
   171  
   172  	defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
   173  	os.Setenv("GOROOT", goroot_bootstrap)
   174  
   175  	defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
   176  	os.Setenv("GOPATH", workspace)
   177  
   178  	defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
   179  	os.Setenv("GOBIN", "")
   180  
   181  	os.Setenv("GOOS", "")
   182  	os.Setenv("GOHOSTOS", "")
   183  	os.Setenv("GOARCH", "")
   184  	os.Setenv("GOHOSTARCH", "")
   185  
   186  	// Run Go 1.4 to build binaries. Use -gcflags=-l to disable inlining to
   187  	// workaround bugs in Go 1.4's compiler. See discussion thread:
   188  	// https://groups.google.com/d/msg/golang-dev/Ss7mCKsvk8w/Gsq7VYI0AwAJ
   189  	// Use the math_big_pure_go build tag to disable the assembly in math/big
   190  	// which may contain unsupported instructions.
   191  	// Note that if we are using Go 1.10 or later as bootstrap, the -gcflags=-l
   192  	// only applies to the final cmd/go binary, but that's OK: if this is Go 1.10
   193  	// or later we don't need to disable inlining to work around bugs in the Go 1.4 compiler.
   194  	cmd := []string{
   195  		pathf("%s/bin/go", goroot_bootstrap),
   196  		"install",
   197  		"-gcflags=-l",
   198  		"-tags=math_big_pure_go compiler_bootstrap",
   199  	}
   200  	if vflag > 0 {
   201  		cmd = append(cmd, "-v")
   202  	}
   203  	if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
   204  		cmd = append(cmd, "-toolexec="+tool)
   205  	}
   206  	cmd = append(cmd, "bootstrap/cmd/...")
   207  	run(base, ShowOutput|CheckExit, cmd...)
   208  
   209  	// Copy binaries into tool binary directory.
   210  	for _, name := range bootstrapDirs {
   211  		if !strings.HasPrefix(name, "cmd/") {
   212  			continue
   213  		}
   214  		name = name[len("cmd/"):]
   215  		if !strings.Contains(name, "/") {
   216  			copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
   217  		}
   218  	}
   219  
   220  	if vflag > 0 {
   221  		xprintf("\n")
   222  	}
   223  }
   224  
   225  var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
   226  
   227  // isUnneededSSARewriteFile reports whether srcFile is a
   228  // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
   229  // architecture that isn't for the current runtime.GOARCH.
   230  //
   231  // When unneeded is true archCaps is the rewrite base filename without
   232  // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
   233  func isUnneededSSARewriteFile(srcFile string) (archCaps string, unneeded bool) {
   234  	if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
   235  		return "", false
   236  	}
   237  	fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
   238  	if fileArch == "" {
   239  		return "", false
   240  	}
   241  	b := fileArch[0]
   242  	if b == '_' || ('a' <= b && b <= 'z') {
   243  		return "", false
   244  	}
   245  	archCaps = fileArch
   246  	fileArch = strings.ToLower(fileArch)
   247  	fileArch = strings.TrimSuffix(fileArch, "splitload")
   248  	if fileArch == os.Getenv("GOHOSTARCH") {
   249  		return "", false
   250  	}
   251  	if fileArch == strings.TrimSuffix(runtime.GOARCH, "le") {
   252  		return "", false
   253  	}
   254  	if fileArch == strings.TrimSuffix(os.Getenv("GOARCH"), "le") {
   255  		return "", false
   256  	}
   257  	return archCaps, true
   258  }
   259  
   260  func bootstrapRewriteFile(srcFile string) string {
   261  	// During bootstrap, generate dummy rewrite files for
   262  	// irrelevant architectures. We only need to build a bootstrap
   263  	// binary that works for the current runtime.GOARCH.
   264  	// This saves 6+ seconds of bootstrap.
   265  	if archCaps, ok := isUnneededSSARewriteFile(srcFile); ok {
   266  		return fmt.Sprintf(`// Code generated by go tool dist; DO NOT EDIT.
   267  
   268  package ssa
   269  
   270  func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
   271  func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
   272  `, archCaps, archCaps)
   273  	}
   274  
   275  	return bootstrapFixImports(srcFile)
   276  }
   277  
   278  func bootstrapFixImports(srcFile string) string {
   279  	lines := strings.SplitAfter(readfile(srcFile), "\n")
   280  	inBlock := false
   281  	for i, line := range lines {
   282  		if strings.HasPrefix(line, "import (") {
   283  			inBlock = true
   284  			continue
   285  		}
   286  		if inBlock && strings.HasPrefix(line, ")") {
   287  			inBlock = false
   288  			continue
   289  		}
   290  		if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
   291  			inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"") || strings.HasPrefix(line, "\texec \"")) {
   292  			line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
   293  			// During bootstrap, must use plain os/exec.
   294  			line = strings.Replace(line, `exec "internal/execabs"`, `"os/exec"`, -1)
   295  			for _, dir := range bootstrapDirs {
   296  				if strings.HasPrefix(dir, "cmd/") {
   297  					continue
   298  				}
   299  				line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
   300  			}
   301  			lines[i] = line
   302  		}
   303  	}
   304  
   305  	lines[0] = "// Code generated by go tool dist; DO NOT EDIT.\n// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
   306  
   307  	return strings.Join(lines, "")
   308  }
   309
View as plain text