generate.go

Documentation: cmd/go/internal/generate

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package generate implements the ``go generate'' command.
     6  package generate
     7  
     8  import (
     9  	"bufio"
    10  	"bytes"
    11  	"context"
    12  	"fmt"
    13  	"go/parser"
    14  	"go/token"
    15  	exec "internal/execabs"
    16  	"io"
    17  	"log"
    18  	"os"
    19  	"path/filepath"
    20  	"regexp"
    21  	"strconv"
    22  	"strings"
    23  
    24  	"cmd/go/internal/base"
    25  	"cmd/go/internal/cfg"
    26  	"cmd/go/internal/load"
    27  	"cmd/go/internal/modload"
    28  	"cmd/go/internal/str"
    29  	"cmd/go/internal/work"
    30  )
    31  
    32  var CmdGenerate = &base.Command{
    33  	Run:       runGenerate,
    34  	UsageLine: "go generate [-run regexp] [-n] [-v] [-x] [build flags] [file.go... | packages]",
    35  	Short:     "generate Go files by processing source",
    36  	Long: `
    37  Generate runs commands described by directives within existing
    38  files. Those commands can run any process but the intent is to
    39  create or update Go source files.
    40  
    41  Go generate is never run automatically by go build, go get, go test,
    42  and so on. It must be run explicitly.
    43  
    44  Go generate scans the file for directives, which are lines of
    45  the form,
    46  
    47  	//go:generate command argument...
    48  
    49  (note: no leading spaces and no space in "//go") where command
    50  is the generator to be run, corresponding to an executable file
    51  that can be run locally. It must either be in the shell path
    52  (gofmt), a fully qualified path (/usr/you/bin/mytool), or a
    53  command alias, described below.
    54  
    55  Note that go generate does not parse the file, so lines that look
    56  like directives in comments or multiline strings will be treated
    57  as directives.
    58  
    59  The arguments to the directive are space-separated tokens or
    60  double-quoted strings passed to the generator as individual
    61  arguments when it is run.
    62  
    63  Quoted strings use Go syntax and are evaluated before execution; a
    64  quoted string appears as a single argument to the generator.
    65  
    66  To convey to humans and machine tools that code is generated,
    67  generated source should have a line that matches the following
    68  regular expression (in Go syntax):
    69  
    70  	^// Code generated .* DO NOT EDIT\.$
    71  
    72  This line must appear before the first non-comment, non-blank
    73  text in the file.
    74  
    75  Go generate sets several variables when it runs the generator:
    76  
    77  	$GOARCH
    78  		The execution architecture (arm, amd64, etc.)
    79  	$GOOS
    80  		The execution operating system (linux, windows, etc.)
    81  	$GOFILE
    82  		The base name of the file.
    83  	$GOLINE
    84  		The line number of the directive in the source file.
    85  	$GOPACKAGE
    86  		The name of the package of the file containing the directive.
    87  	$DOLLAR
    88  		A dollar sign.
    89  
    90  Other than variable substitution and quoted-string evaluation, no
    91  special processing such as "globbing" is performed on the command
    92  line.
    93  
    94  As a last step before running the command, any invocations of any
    95  environment variables with alphanumeric names, such as $GOFILE or
    96  $HOME, are expanded throughout the command line. The syntax for
    97  variable expansion is $NAME on all operating systems. Due to the
    98  order of evaluation, variables are expanded even inside quoted
    99  strings. If the variable NAME is not set, $NAME expands to the
   100  empty string.
   101  
   102  A directive of the form,
   103  
   104  	//go:generate -command xxx args...
   105  
   106  specifies, for the remainder of this source file only, that the
   107  string xxx represents the command identified by the arguments. This
   108  can be used to create aliases or to handle multiword generators.
   109  For example,
   110  
   111  	//go:generate -command foo go tool foo
   112  
   113  specifies that the command "foo" represents the generator
   114  "go tool foo".
   115  
   116  Generate processes packages in the order given on the command line,
   117  one at a time. If the command line lists .go files from a single directory,
   118  they are treated as a single package. Within a package, generate processes the
   119  source files in a package in file name order, one at a time. Within
   120  a source file, generate runs generators in the order they appear
   121  in the file, one at a time. The go generate tool also sets the build
   122  tag "generate" so that files may be examined by go generate but ignored
   123  during build.
   124  
   125  For packages with invalid code, generate processes only source files with a
   126  valid package clause.
   127  
   128  If any generator returns an error exit status, "go generate" skips
   129  all further processing for that package.
   130  
   131  The generator is run in the package's source directory.
   132  
   133  Go generate accepts one specific flag:
   134  
   135  	-run=""
   136  		if non-empty, specifies a regular expression to select
   137  		directives whose full original source text (excluding
   138  		any trailing spaces and final newline) matches the
   139  		expression.
   140  
   141  It also accepts the standard build flags including -v, -n, and -x.
   142  The -v flag prints the names of packages and files as they are
   143  processed.
   144  The -n flag prints commands that would be executed.
   145  The -x flag prints commands as they are executed.
   146  
   147  For more about build flags, see 'go help build'.
   148  
   149  For more about specifying packages, see 'go help packages'.
   150  	`,
   151  }
   152  
   153  var (
   154  	generateRunFlag string         // generate -run flag
   155  	generateRunRE   *regexp.Regexp // compiled expression for -run
   156  )
   157  
   158  func init() {
   159  	work.AddBuildFlags(CmdGenerate, work.DefaultBuildFlags)
   160  	CmdGenerate.Flag.StringVar(&generateRunFlag, "run", "", "")
   161  }
   162  
   163  func runGenerate(ctx context.Context, cmd *base.Command, args []string) {
   164  	if generateRunFlag != "" {
   165  		var err error
   166  		generateRunRE, err = regexp.Compile(generateRunFlag)
   167  		if err != nil {
   168  			log.Fatalf("generate: %s", err)
   169  		}
   170  	}
   171  
   172  	cfg.BuildContext.BuildTags = append(cfg.BuildContext.BuildTags, "generate")
   173  
   174  	// Even if the arguments are .go files, this loop suffices.
   175  	printed := false
   176  	pkgOpts := load.PackageOpts{IgnoreImports: true}
   177  	for _, pkg := range load.PackagesAndErrors(ctx, pkgOpts, args) {
   178  		if modload.Enabled() && pkg.Module != nil && !pkg.Module.Main {
   179  			if !printed {
   180  				fmt.Fprintf(os.Stderr, "go: not generating in packages in dependency modules\n")
   181  				printed = true
   182  			}
   183  			continue
   184  		}
   185  
   186  		for _, file := range pkg.InternalGoFiles() {
   187  			if !generate(file) {
   188  				break
   189  			}
   190  		}
   191  
   192  		for _, file := range pkg.InternalXGoFiles() {
   193  			if !generate(file) {
   194  				break
   195  			}
   196  		}
   197  	}
   198  }
   199  
   200  // generate runs the generation directives for a single file.
   201  func generate(absFile string) bool {
   202  	src, err := os.ReadFile(absFile)
   203  	if err != nil {
   204  		log.Fatalf("generate: %s", err)
   205  	}
   206  
   207  	// Parse package clause
   208  	filePkg, err := parser.ParseFile(token.NewFileSet(), "", src, parser.PackageClauseOnly)
   209  	if err != nil {
   210  		// Invalid package clause - ignore file.
   211  		return true
   212  	}
   213  
   214  	g := &Generator{
   215  		r:        bytes.NewReader(src),
   216  		path:     absFile,
   217  		pkg:      filePkg.Name.String(),
   218  		commands: make(map[string][]string),
   219  	}
   220  	return g.run()
   221  }
   222  
   223  // A Generator represents the state of a single Go source file
   224  // being scanned for generator commands.
   225  type Generator struct {
   226  	r        io.Reader
   227  	path     string // full rooted path name.
   228  	dir      string // full rooted directory of file.
   229  	file     string // base name of file.
   230  	pkg      string
   231  	commands map[string][]string
   232  	lineNum  int // current line number.
   233  	env      []string
   234  }
   235  
   236  // run runs the generators in the current file.
   237  func (g *Generator) run() (ok bool) {
   238  	// Processing below here calls g.errorf on failure, which does panic(stop).
   239  	// If we encounter an error, we abort the package.
   240  	defer func() {
   241  		e := recover()
   242  		if e != nil {
   243  			ok = false
   244  			if e != stop {
   245  				panic(e)
   246  			}
   247  			base.SetExitStatus(1)
   248  		}
   249  	}()
   250  	g.dir, g.file = filepath.Split(g.path)
   251  	g.dir = filepath.Clean(g.dir) // No final separator please.
   252  	if cfg.BuildV {
   253  		fmt.Fprintf(os.Stderr, "%s\n", base.ShortPath(g.path))
   254  	}
   255  
   256  	// Scan for lines that start "//go:generate".
   257  	// Can't use bufio.Scanner because it can't handle long lines,
   258  	// which are likely to appear when using generate.
   259  	input := bufio.NewReader(g.r)
   260  	var err error
   261  	// One line per loop.
   262  	for {
   263  		g.lineNum++ // 1-indexed.
   264  		var buf []byte
   265  		buf, err = input.ReadSlice('\n')
   266  		if err == bufio.ErrBufferFull {
   267  			// Line too long - consume and ignore.
   268  			if isGoGenerate(buf) {
   269  				g.errorf("directive too long")
   270  			}
   271  			for err == bufio.ErrBufferFull {
   272  				_, err = input.ReadSlice('\n')
   273  			}
   274  			if err != nil {
   275  				break
   276  			}
   277  			continue
   278  		}
   279  
   280  		if err != nil {
   281  			// Check for marker at EOF without final \n.
   282  			if err == io.EOF && isGoGenerate(buf) {
   283  				err = io.ErrUnexpectedEOF
   284  			}
   285  			break
   286  		}
   287  
   288  		if !isGoGenerate(buf) {
   289  			continue
   290  		}
   291  		if generateRunFlag != "" {
   292  			if !generateRunRE.Match(bytes.TrimSpace(buf)) {
   293  				continue
   294  			}
   295  		}
   296  
   297  		g.setEnv()
   298  		words := g.split(string(buf))
   299  		if len(words) == 0 {
   300  			g.errorf("no arguments to directive")
   301  		}
   302  		if words[0] == "-command" {
   303  			g.setShorthand(words)
   304  			continue
   305  		}
   306  		// Run the command line.
   307  		if cfg.BuildN || cfg.BuildX {
   308  			fmt.Fprintf(os.Stderr, "%s\n", strings.Join(words, " "))
   309  		}
   310  		if cfg.BuildN {
   311  			continue
   312  		}
   313  		g.exec(words)
   314  	}
   315  	if err != nil && err != io.EOF {
   316  		g.errorf("error reading %s: %s", base.ShortPath(g.path), err)
   317  	}
   318  	return true
   319  }
   320  
   321  func isGoGenerate(buf []byte) bool {
   322  	return bytes.HasPrefix(buf, []byte("//go:generate ")) || bytes.HasPrefix(buf, []byte("//go:generate\t"))
   323  }
   324  
   325  // setEnv sets the extra environment variables used when executing a
   326  // single go:generate command.
   327  func (g *Generator) setEnv() {
   328  	g.env = []string{
   329  		"GOARCH=" + cfg.BuildContext.GOARCH,
   330  		"GOOS=" + cfg.BuildContext.GOOS,
   331  		"GOFILE=" + g.file,
   332  		"GOLINE=" + strconv.Itoa(g.lineNum),
   333  		"GOPACKAGE=" + g.pkg,
   334  		"DOLLAR=" + "$",
   335  	}
   336  	g.env = base.AppendPWD(g.env, g.dir)
   337  }
   338  
   339  // split breaks the line into words, evaluating quoted
   340  // strings and evaluating environment variables.
   341  // The initial //go:generate element is present in line.
   342  func (g *Generator) split(line string) []string {
   343  	// Parse line, obeying quoted strings.
   344  	var words []string
   345  	line = line[len("//go:generate ") : len(line)-1] // Drop preamble and final newline.
   346  	// There may still be a carriage return.
   347  	if len(line) > 0 && line[len(line)-1] == '\r' {
   348  		line = line[:len(line)-1]
   349  	}
   350  	// One (possibly quoted) word per iteration.
   351  Words:
   352  	for {
   353  		line = strings.TrimLeft(line, " \t")
   354  		if len(line) == 0 {
   355  			break
   356  		}
   357  		if line[0] == '"' {
   358  			for i := 1; i < len(line); i++ {
   359  				c := line[i] // Only looking for ASCII so this is OK.
   360  				switch c {
   361  				case '\\':
   362  					if i+1 == len(line) {
   363  						g.errorf("bad backslash")
   364  					}
   365  					i++ // Absorb next byte (If it's a multibyte we'll get an error in Unquote).
   366  				case '"':
   367  					word, err := strconv.Unquote(line[0 : i+1])
   368  					if err != nil {
   369  						g.errorf("bad quoted string")
   370  					}
   371  					words = append(words, word)
   372  					line = line[i+1:]
   373  					// Check the next character is space or end of line.
   374  					if len(line) > 0 && line[0] != ' ' && line[0] != '\t' {
   375  						g.errorf("expect space after quoted argument")
   376  					}
   377  					continue Words
   378  				}
   379  			}
   380  			g.errorf("mismatched quoted string")
   381  		}
   382  		i := strings.IndexAny(line, " \t")
   383  		if i < 0 {
   384  			i = len(line)
   385  		}
   386  		words = append(words, line[0:i])
   387  		line = line[i:]
   388  	}
   389  	// Substitute command if required.
   390  	if len(words) > 0 && g.commands[words[0]] != nil {
   391  		// Replace 0th word by command substitution.
   392  		//
   393  		// Force a copy of the command definition to
   394  		// ensure words doesn't end up as a reference
   395  		// to the g.commands content.
   396  		tmpCmdWords := append([]string(nil), (g.commands[words[0]])...)
   397  		words = append(tmpCmdWords, words[1:]...)
   398  	}
   399  	// Substitute environment variables.
   400  	for i, word := range words {
   401  		words[i] = os.Expand(word, g.expandVar)
   402  	}
   403  	return words
   404  }
   405  
   406  var stop = fmt.Errorf("error in generation")
   407  
   408  // errorf logs an error message prefixed with the file and line number.
   409  // It then exits the program (with exit status 1) because generation stops
   410  // at the first error.
   411  func (g *Generator) errorf(format string, args ...interface{}) {
   412  	fmt.Fprintf(os.Stderr, "%s:%d: %s\n", base.ShortPath(g.path), g.lineNum,
   413  		fmt.Sprintf(format, args...))
   414  	panic(stop)
   415  }
   416  
   417  // expandVar expands the $XXX invocation in word. It is called
   418  // by os.Expand.
   419  func (g *Generator) expandVar(word string) string {
   420  	w := word + "="
   421  	for _, e := range g.env {
   422  		if strings.HasPrefix(e, w) {
   423  			return e[len(w):]
   424  		}
   425  	}
   426  	return os.Getenv(word)
   427  }
   428  
   429  // setShorthand installs a new shorthand as defined by a -command directive.
   430  func (g *Generator) setShorthand(words []string) {
   431  	// Create command shorthand.
   432  	if len(words) == 1 {
   433  		g.errorf("no command specified for -command")
   434  	}
   435  	command := words[1]
   436  	if g.commands[command] != nil {
   437  		g.errorf("command %q multiply defined", command)
   438  	}
   439  	g.commands[command] = words[2:len(words):len(words)] // force later append to make copy
   440  }
   441  
   442  // exec runs the command specified by the argument. The first word is
   443  // the command name itself.
   444  func (g *Generator) exec(words []string) {
   445  	cmd := exec.Command(words[0], words[1:]...)
   446  	// Standard in and out of generator should be the usual.
   447  	cmd.Stdout = os.Stdout
   448  	cmd.Stderr = os.Stderr
   449  	// Run the command in the package directory.
   450  	cmd.Dir = g.dir
   451  	cmd.Env = str.StringList(cfg.OrigEnv, g.env)
   452  	err := cmd.Run()
   453  	if err != nil {
   454  		g.errorf("running %q: %s", words[0], err)
   455  	}
   456  }
   457
View as plain text