Black Lives Matter. Support the Equal Justice Initiative.

Source file src/go/scanner/scanner.go

Documentation: go/scanner

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package scanner implements a scanner for Go source text.
     6  // It takes a []byte as source which can then be tokenized
     7  // through repeated calls to the Scan method.
     8  //
     9  package scanner
    10  
    11  import (
    12  	"bytes"
    13  	"fmt"
    14  	"go/token"
    15  	"path/filepath"
    16  	"strconv"
    17  	"unicode"
    18  	"unicode/utf8"
    19  )
    20  
    21  // An ErrorHandler may be provided to Scanner.Init. If a syntax error is
    22  // encountered and a handler was installed, the handler is called with a
    23  // position and an error message. The position points to the beginning of
    24  // the offending token.
    25  //
    26  type ErrorHandler func(pos token.Position, msg string)
    27  
    28  // A Scanner holds the scanner's internal state while processing
    29  // a given text. It can be allocated as part of another data
    30  // structure but must be initialized via Init before use.
    31  //
    32  type Scanner struct {
    33  	// immutable state
    34  	file *token.File  // source file handle
    35  	dir  string       // directory portion of file.Name()
    36  	src  []byte       // source
    37  	err  ErrorHandler // error reporting; or nil
    38  	mode Mode         // scanning mode
    39  
    40  	// scanning state
    41  	ch         rune // current character
    42  	offset     int  // character offset
    43  	rdOffset   int  // reading offset (position after current character)
    44  	lineOffset int  // current line offset
    45  	insertSemi bool // insert a semicolon before next newline
    46  
    47  	// public state - ok to modify
    48  	ErrorCount int // number of errors encountered
    49  }
    50  
    51  const (
    52  	bom = 0xFEFF // byte order mark, only permitted as very first character
    53  	eof = -1     // end of file
    54  )
    55  
    56  // Read the next Unicode char into s.ch.
    57  // s.ch < 0 means end-of-file.
    58  //
    59  // For optimization, there is some overlap between this method and
    60  // s.scanIdentifier.
    61  func (s *Scanner) next() {
    62  	if s.rdOffset < len(s.src) {
    63  		s.offset = s.rdOffset
    64  		if s.ch == '\n' {
    65  			s.lineOffset = s.offset
    66  			s.file.AddLine(s.offset)
    67  		}
    68  		r, w := rune(s.src[s.rdOffset]), 1
    69  		switch {
    70  		case r == 0:
    71  			s.error(s.offset, "illegal character NUL")
    72  		case r >= utf8.RuneSelf:
    73  			// not ASCII
    74  			r, w = utf8.DecodeRune(s.src[s.rdOffset:])
    75  			if r == utf8.RuneError && w == 1 {
    76  				s.error(s.offset, "illegal UTF-8 encoding")
    77  			} else if r == bom && s.offset > 0 {
    78  				s.error(s.offset, "illegal byte order mark")
    79  			}
    80  		}
    81  		s.rdOffset += w
    82  		s.ch = r
    83  	} else {
    84  		s.offset = len(s.src)
    85  		if s.ch == '\n' {
    86  			s.lineOffset = s.offset
    87  			s.file.AddLine(s.offset)
    88  		}
    89  		s.ch = eof
    90  	}
    91  }
    92  
    93  // peek returns the byte following the most recently read character without
    94  // advancing the scanner. If the scanner is at EOF, peek returns 0.
    95  func (s *Scanner) peek() byte {
    96  	if s.rdOffset < len(s.src) {
    97  		return s.src[s.rdOffset]
    98  	}
    99  	return 0
   100  }
   101  
   102  // A mode value is a set of flags (or 0).
   103  // They control scanner behavior.
   104  //
   105  type Mode uint
   106  
   107  const (
   108  	ScanComments    Mode = 1 << iota // return comments as COMMENT tokens
   109  	dontInsertSemis                  // do not automatically insert semicolons - for testing only
   110  )
   111  
   112  // Init prepares the scanner s to tokenize the text src by setting the
   113  // scanner at the beginning of src. The scanner uses the file set file
   114  // for position information and it adds line information for each line.
   115  // It is ok to re-use the same file when re-scanning the same file as
   116  // line information which is already present is ignored. Init causes a
   117  // panic if the file size does not match the src size.
   118  //
   119  // Calls to Scan will invoke the error handler err if they encounter a
   120  // syntax error and err is not nil. Also, for each error encountered,
   121  // the Scanner field ErrorCount is incremented by one. The mode parameter
   122  // determines how comments are handled.
   123  //
   124  // Note that Init may call err if there is an error in the first character
   125  // of the file.
   126  //
   127  func (s *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode Mode) {
   128  	// Explicitly initialize all fields since a scanner may be reused.
   129  	if file.Size() != len(src) {
   130  		panic(fmt.Sprintf("file size (%d) does not match src len (%d)", file.Size(), len(src)))
   131  	}
   132  	s.file = file
   133  	s.dir, _ = filepath.Split(file.Name())
   134  	s.src = src
   135  	s.err = err
   136  	s.mode = mode
   137  
   138  	s.ch = ' '
   139  	s.offset = 0
   140  	s.rdOffset = 0
   141  	s.lineOffset = 0
   142  	s.insertSemi = false
   143  	s.ErrorCount = 0
   144  
   145  	s.next()
   146  	if s.ch == bom {
   147  		s.next() // ignore BOM at file beginning
   148  	}
   149  }
   150  
   151  func (s *Scanner) error(offs int, msg string) {
   152  	if s.err != nil {
   153  		s.err(s.file.Position(s.file.Pos(offs)), msg)
   154  	}
   155  	s.ErrorCount++
   156  }
   157  
   158  func (s *Scanner) errorf(offs int, format string, args ...interface{}) {
   159  	s.error(offs, fmt.Sprintf(format, args...))
   160  }
   161  
   162  func (s *Scanner) scanComment() string {
   163  	// initial '/' already consumed; s.ch == '/' || s.ch == '*'
   164  	offs := s.offset - 1 // position of initial '/'
   165  	next := -1           // position immediately following the comment; < 0 means invalid comment
   166  	numCR := 0
   167  
   168  	if s.ch == '/' {
   169  		//-style comment
   170  		// (the final '\n' is not considered part of the comment)
   171  		s.next()
   172  		for s.ch != '\n' && s.ch >= 0 {
   173  			if s.ch == '\r' {
   174  				numCR++
   175  			}
   176  			s.next()
   177  		}
   178  		// if we are at '\n', the position following the comment is afterwards
   179  		next = s.offset
   180  		if s.ch == '\n' {
   181  			next++
   182  		}
   183  		goto exit
   184  	}
   185  
   186  	/*-style comment */
   187  	s.next()
   188  	for s.ch >= 0 {
   189  		ch := s.ch
   190  		if ch == '\r' {
   191  			numCR++
   192  		}
   193  		s.next()
   194  		if ch == '*' && s.ch == '/' {
   195  			s.next()
   196  			next = s.offset
   197  			goto exit
   198  		}
   199  	}
   200  
   201  	s.error(offs, "comment not terminated")
   202  
   203  exit:
   204  	lit := s.src[offs:s.offset]
   205  
   206  	// On Windows, a (//-comment) line may end in "\r\n".
   207  	// Remove the final '\r' before analyzing the text for
   208  	// line directives (matching the compiler). Remove any
   209  	// other '\r' afterwards (matching the pre-existing be-
   210  	// havior of the scanner).
   211  	if numCR > 0 && len(lit) >= 2 && lit[1] == '/' && lit[len(lit)-1] == '\r' {
   212  		lit = lit[:len(lit)-1]
   213  		numCR--
   214  	}
   215  
   216  	// interpret line directives
   217  	// (//line directives must start at the beginning of the current line)
   218  	if next >= 0 /* implies valid comment */ && (lit[1] == '*' || offs == s.lineOffset) && bytes.HasPrefix(lit[2:], prefix) {
   219  		s.updateLineInfo(next, offs, lit)
   220  	}
   221  
   222  	if numCR > 0 {
   223  		lit = stripCR(lit, lit[1] == '*')
   224  	}
   225  
   226  	return string(lit)
   227  }
   228  
   229  var prefix = []byte("line ")
   230  
   231  // updateLineInfo parses the incoming comment text at offset offs
   232  // as a line directive. If successful, it updates the line info table
   233  // for the position next per the line directive.
   234  func (s *Scanner) updateLineInfo(next, offs int, text []byte) {
   235  	// extract comment text
   236  	if text[1] == '*' {
   237  		text = text[:len(text)-2] // lop off trailing "*/"
   238  	}
   239  	text = text[7:] // lop off leading "//line " or "/*line "
   240  	offs += 7
   241  
   242  	i, n, ok := trailingDigits(text)
   243  	if i == 0 {
   244  		return // ignore (not a line directive)
   245  	}
   246  	// i > 0
   247  
   248  	if !ok {
   249  		// text has a suffix :xxx but xxx is not a number
   250  		s.error(offs+i, "invalid line number: "+string(text[i:]))
   251  		return
   252  	}
   253  
   254  	var line, col int
   255  	i2, n2, ok2 := trailingDigits(text[:i-1])
   256  	if ok2 {
   257  		//line filename:line:col
   258  		i, i2 = i2, i
   259  		line, col = n2, n
   260  		if col == 0 {
   261  			s.error(offs+i2, "invalid column number: "+string(text[i2:]))
   262  			return
   263  		}
   264  		text = text[:i2-1] // lop off ":col"
   265  	} else {
   266  		//line filename:line
   267  		line = n
   268  	}
   269  
   270  	if line == 0 {
   271  		s.error(offs+i, "invalid line number: "+string(text[i:]))
   272  		return
   273  	}
   274  
   275  	// If we have a column (//line filename:line:col form),
   276  	// an empty filename means to use the previous filename.
   277  	filename := string(text[:i-1]) // lop off ":line", and trim white space
   278  	if filename == "" && ok2 {
   279  		filename = s.file.Position(s.file.Pos(offs)).Filename
   280  	} else if filename != "" {
   281  		// Put a relative filename in the current directory.
   282  		// This is for compatibility with earlier releases.
   283  		// See issue 26671.
   284  		filename = filepath.Clean(filename)
   285  		if !filepath.IsAbs(filename) {
   286  			filename = filepath.Join(s.dir, filename)
   287  		}
   288  	}
   289  
   290  	s.file.AddLineColumnInfo(next, filename, line, col)
   291  }
   292  
   293  func trailingDigits(text []byte) (int, int, bool) {
   294  	i := bytes.LastIndexByte(text, ':') // look from right (Windows filenames may contain ':')
   295  	if i < 0 {
   296  		return 0, 0, false // no ":"
   297  	}
   298  	// i >= 0
   299  	n, err := strconv.ParseUint(string(text[i+1:]), 10, 0)
   300  	return i + 1, int(n), err == nil
   301  }
   302  
   303  func (s *Scanner) findLineEnd() bool {
   304  	// initial '/' already consumed
   305  
   306  	defer func(offs int) {
   307  		// reset scanner state to where it was upon calling findLineEnd
   308  		s.ch = '/'
   309  		s.offset = offs
   310  		s.rdOffset = offs + 1
   311  		s.next() // consume initial '/' again
   312  	}(s.offset - 1)
   313  
   314  	// read ahead until a newline, EOF, or non-comment token is found
   315  	for s.ch == '/' || s.ch == '*' {
   316  		if s.ch == '/' {
   317  			//-style comment always contains a newline
   318  			return true
   319  		}
   320  		/*-style comment: look for newline */
   321  		s.next()
   322  		for s.ch >= 0 {
   323  			ch := s.ch
   324  			if ch == '\n' {
   325  				return true
   326  			}
   327  			s.next()
   328  			if ch == '*' && s.ch == '/' {
   329  				s.next()
   330  				break
   331  			}
   332  		}
   333  		s.skipWhitespace() // s.insertSemi is set
   334  		if s.ch < 0 || s.ch == '\n' {
   335  			return true
   336  		}
   337  		if s.ch != '/' {
   338  			// non-comment token
   339  			return false
   340  		}
   341  		s.next() // consume '/'
   342  	}
   343  
   344  	return false
   345  }
   346  
   347  func isLetter(ch rune) bool {
   348  	return 'a' <= lower(ch) && lower(ch) <= 'z' || ch == '_' || ch >= utf8.RuneSelf && unicode.IsLetter(ch)
   349  }
   350  
   351  func isDigit(ch rune) bool {
   352  	return isDecimal(ch) || ch >= utf8.RuneSelf && unicode.IsDigit(ch)
   353  }
   354  
   355  // scanIdentifier reads the string of valid identifier characters at s.offset.
   356  // It must only be called when s.ch is known to be a valid letter.
   357  //
   358  // Be careful when making changes to this function: it is optimized and affects
   359  // scanning performance significantly.
   360  func (s *Scanner) scanIdentifier() string {
   361  	offs := s.offset
   362  
   363  	// Optimize for the common case of an ASCII identifier.
   364  	//
   365  	// Ranging over s.src[s.rdOffset:] lets us avoid some bounds checks, and
   366  	// avoids conversions to runes.
   367  	//
   368  	// In case we encounter a non-ASCII character, fall back on the slower path
   369  	// of calling into s.next().
   370  	for rdOffset, b := range s.src[s.rdOffset:] {
   371  		if 'a' <= b && b <= 'z' || 'A' <= b && b <= 'Z' || b == '_' || '0' <= b && b <= '9' {
   372  			// Avoid assigning a rune for the common case of an ascii character.
   373  			continue
   374  		}
   375  		s.rdOffset += rdOffset
   376  		if 0 < b && b < utf8.RuneSelf {
   377  			// Optimization: we've encountered an ASCII character that's not a letter
   378  			// or number. Avoid the call into s.next() and corresponding set up.
   379  			//
   380  			// Note that s.next() does some line accounting if s.ch is '\n', so this
   381  			// shortcut is only possible because we know that the preceding character
   382  			// is not '\n'.
   383  			s.ch = rune(b)
   384  			s.offset = s.rdOffset
   385  			s.rdOffset++
   386  			goto exit
   387  		}
   388  		// We know that the preceding character is valid for an identifier because
   389  		// scanIdentifier is only called when s.ch is a letter, so calling s.next()
   390  		// at s.rdOffset resets the scanner state.
   391  		s.next()
   392  		for isLetter(s.ch) || isDigit(s.ch) {
   393  			s.next()
   394  		}
   395  		goto exit
   396  	}
   397  	s.offset = len(s.src)
   398  	s.rdOffset = len(s.src)
   399  	s.ch = eof
   400  
   401  exit:
   402  	return string(s.src[offs:s.offset])
   403  }
   404  
   405  func digitVal(ch rune) int {
   406  	switch {
   407  	case '0' <= ch && ch <= '9':
   408  		return int(ch - '0')
   409  	case 'a' <= lower(ch) && lower(ch) <= 'f':
   410  		return int(lower(ch) - 'a' + 10)
   411  	}
   412  	return 16 // larger than any legal digit val
   413  }
   414  
   415  func lower(ch rune) rune     { return ('a' - 'A') | ch } // returns lower-case ch iff ch is ASCII letter
   416  func isDecimal(ch rune) bool { return '0' <= ch && ch <= '9' }
   417  func isHex(ch rune) bool     { return '0' <= ch && ch <= '9' || 'a' <= lower(ch) && lower(ch) <= 'f' }
   418  
   419  // digits accepts the sequence { digit | '_' }.
   420  // If base <= 10, digits accepts any decimal digit but records
   421  // the offset (relative to the source start) of a digit >= base
   422  // in *invalid, if *invalid < 0.
   423  // digits returns a bitset describing whether the sequence contained
   424  // digits (bit 0 is set), or separators '_' (bit 1 is set).
   425  func (s *Scanner) digits(base int, invalid *int) (digsep int) {
   426  	if base <= 10 {
   427  		max := rune('0' + base)
   428  		for isDecimal(s.ch) || s.ch == '_' {
   429  			ds := 1
   430  			if s.ch == '_' {
   431  				ds = 2
   432  			} else if s.ch >= max && *invalid < 0 {
   433  				*invalid = s.offset // record invalid rune offset
   434  			}
   435  			digsep |= ds
   436  			s.next()
   437  		}
   438  	} else {
   439  		for isHex(s.ch) || s.ch == '_' {
   440  			ds := 1
   441  			if s.ch == '_' {
   442  				ds = 2
   443  			}
   444  			digsep |= ds
   445  			s.next()
   446  		}
   447  	}
   448  	return
   449  }
   450  
   451  func (s *Scanner) scanNumber() (token.Token, string) {
   452  	offs := s.offset
   453  	tok := token.ILLEGAL
   454  
   455  	base := 10        // number base
   456  	prefix := rune(0) // one of 0 (decimal), '0' (0-octal), 'x', 'o', or 'b'
   457  	digsep := 0       // bit 0: digit present, bit 1: '_' present
   458  	invalid := -1     // index of invalid digit in literal, or < 0
   459  
   460  	// integer part
   461  	if s.ch != '.' {
   462  		tok = token.INT
   463  		if s.ch == '0' {
   464  			s.next()
   465  			switch lower(s.ch) {
   466  			case 'x':
   467  				s.next()
   468  				base, prefix = 16, 'x'
   469  			case 'o':
   470  				s.next()
   471  				base, prefix = 8, 'o'
   472  			case 'b':
   473  				s.next()
   474  				base, prefix = 2, 'b'
   475  			default:
   476  				base, prefix = 8, '0'
   477  				digsep = 1 // leading 0
   478  			}
   479  		}
   480  		digsep |= s.digits(base, &invalid)
   481  	}
   482  
   483  	// fractional part
   484  	if s.ch == '.' {
   485  		tok = token.FLOAT
   486  		if prefix == 'o' || prefix == 'b' {
   487  			s.error(s.offset, "invalid radix point in "+litname(prefix))
   488  		}
   489  		s.next()
   490  		digsep |= s.digits(base, &invalid)
   491  	}
   492  
   493  	if digsep&1 == 0 {
   494  		s.error(s.offset, litname(prefix)+" has no digits")
   495  	}
   496  
   497  	// exponent
   498  	if e := lower(s.ch); e == 'e' || e == 'p' {
   499  		switch {
   500  		case e == 'e' && prefix != 0 && prefix != '0':
   501  			s.errorf(s.offset, "%q exponent requires decimal mantissa", s.ch)
   502  		case e == 'p' && prefix != 'x':
   503  			s.errorf(s.offset, "%q exponent requires hexadecimal mantissa", s.ch)
   504  		}
   505  		s.next()
   506  		tok = token.FLOAT
   507  		if s.ch == '+' || s.ch == '-' {
   508  			s.next()
   509  		}
   510  		ds := s.digits(10, nil)
   511  		digsep |= ds
   512  		if ds&1 == 0 {
   513  			s.error(s.offset, "exponent has no digits")
   514  		}
   515  	} else if prefix == 'x' && tok == token.FLOAT {
   516  		s.error(s.offset, "hexadecimal mantissa requires a 'p' exponent")
   517  	}
   518  
   519  	// suffix 'i'
   520  	if s.ch == 'i' {
   521  		tok = token.IMAG
   522  		s.next()
   523  	}
   524  
   525  	lit := string(s.src[offs:s.offset])
   526  	if tok == token.INT && invalid >= 0 {
   527  		s.errorf(invalid, "invalid digit %q in %s", lit[invalid-offs], litname(prefix))
   528  	}
   529  	if digsep&2 != 0 {
   530  		if i := invalidSep(lit); i >= 0 {
   531  			s.error(offs+i, "'_' must separate successive digits")
   532  		}
   533  	}
   534  
   535  	return tok, lit
   536  }
   537  
   538  func litname(prefix rune) string {
   539  	switch prefix {
   540  	case 'x':
   541  		return "hexadecimal literal"
   542  	case 'o', '0':
   543  		return "octal literal"
   544  	case 'b':
   545  		return "binary literal"
   546  	}
   547  	return "decimal literal"
   548  }
   549  
   550  // invalidSep returns the index of the first invalid separator in x, or -1.
   551  func invalidSep(x string) int {
   552  	x1 := ' ' // prefix char, we only care if it's 'x'
   553  	d := '.'  // digit, one of '_', '0' (a digit), or '.' (anything else)
   554  	i := 0
   555  
   556  	// a prefix counts as a digit
   557  	if len(x) >= 2 && x[0] == '0' {
   558  		x1 = lower(rune(x[1]))
   559  		if x1 == 'x' || x1 == 'o' || x1 == 'b' {
   560  			d = '0'
   561  			i = 2
   562  		}
   563  	}
   564  
   565  	// mantissa and exponent
   566  	for ; i < len(x); i++ {
   567  		p := d // previous digit
   568  		d = rune(x[i])
   569  		switch {
   570  		case d == '_':
   571  			if p != '0' {
   572  				return i
   573  			}
   574  		case isDecimal(d) || x1 == 'x' && isHex(d):
   575  			d = '0'
   576  		default:
   577  			if p == '_' {
   578  				return i - 1
   579  			}
   580  			d = '.'
   581  		}
   582  	}
   583  	if d == '_' {
   584  		return len(x) - 1
   585  	}
   586  
   587  	return -1
   588  }
   589  
   590  // scanEscape parses an escape sequence where rune is the accepted
   591  // escaped quote. In case of a syntax error, it stops at the offending
   592  // character (without consuming it) and returns false. Otherwise
   593  // it returns true.
   594  func (s *Scanner) scanEscape(quote rune) bool {
   595  	offs := s.offset
   596  
   597  	var n int
   598  	var base, max uint32
   599  	switch s.ch {
   600  	case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote:
   601  		s.next()
   602  		return true
   603  	case '0', '1', '2', '3', '4', '5', '6', '7':
   604  		n, base, max = 3, 8, 255
   605  	case 'x':
   606  		s.next()
   607  		n, base, max = 2, 16, 255
   608  	case 'u':
   609  		s.next()
   610  		n, base, max = 4, 16, unicode.MaxRune
   611  	case 'U':
   612  		s.next()
   613  		n, base, max = 8, 16, unicode.MaxRune
   614  	default:
   615  		msg := "unknown escape sequence"
   616  		if s.ch < 0 {
   617  			msg = "escape sequence not terminated"
   618  		}
   619  		s.error(offs, msg)
   620  		return false
   621  	}
   622  
   623  	var x uint32
   624  	for n > 0 {
   625  		d := uint32(digitVal(s.ch))
   626  		if d >= base {
   627  			msg := fmt.Sprintf("illegal character %#U in escape sequence", s.ch)
   628  			if s.ch < 0 {
   629  				msg = "escape sequence not terminated"
   630  			}
   631  			s.error(s.offset, msg)
   632  			return false
   633  		}
   634  		x = x*base + d
   635  		s.next()
   636  		n--
   637  	}
   638  
   639  	if x > max || 0xD800 <= x && x < 0xE000 {
   640  		s.error(offs, "escape sequence is invalid Unicode code point")
   641  		return false
   642  	}
   643  
   644  	return true
   645  }
   646  
   647  func (s *Scanner) scanRune() string {
   648  	// '\'' opening already consumed
   649  	offs := s.offset - 1
   650  
   651  	valid := true
   652  	n := 0
   653  	for {
   654  		ch := s.ch
   655  		if ch == '\n' || ch < 0 {
   656  			// only report error if we don't have one already
   657  			if valid {
   658  				s.error(offs, "rune literal not terminated")
   659  				valid = false
   660  			}
   661  			break
   662  		}
   663  		s.next()
   664  		if ch == '\'' {
   665  			break
   666  		}
   667  		n++
   668  		if ch == '\\' {
   669  			if !s.scanEscape('\'') {
   670  				valid = false
   671  			}
   672  			// continue to read to closing quote
   673  		}
   674  	}
   675  
   676  	if valid && n != 1 {
   677  		s.error(offs, "illegal rune literal")
   678  	}
   679  
   680  	return string(s.src[offs:s.offset])
   681  }
   682  
   683  func (s *Scanner) scanString() string {
   684  	// '"' opening already consumed
   685  	offs := s.offset - 1
   686  
   687  	for {
   688  		ch := s.ch
   689  		if ch == '\n' || ch < 0 {
   690  			s.error(offs, "string literal not terminated")
   691  			break
   692  		}
   693  		s.next()
   694  		if ch == '"' {
   695  			break
   696  		}
   697  		if ch == '\\' {
   698  			s.scanEscape('"')
   699  		}
   700  	}
   701  
   702  	return string(s.src[offs:s.offset])
   703  }
   704  
   705  func stripCR(b []byte, comment bool) []byte {
   706  	c := make([]byte, len(b))
   707  	i := 0
   708  	for j, ch := range b {
   709  		// In a /*-style comment, don't strip \r from *\r/ (incl.
   710  		// sequences of \r from *\r\r...\r/) since the resulting
   711  		// */ would terminate the comment too early unless the \r
   712  		// is immediately following the opening /* in which case
   713  		// it's ok because /*/ is not closed yet (issue #11151).
   714  		if ch != '\r' || comment && i > len("/*") && c[i-1] == '*' && j+1 < len(b) && b[j+1] == '/' {
   715  			c[i] = ch
   716  			i++
   717  		}
   718  	}
   719  	return c[:i]
   720  }
   721  
   722  func (s *Scanner) scanRawString() string {
   723  	// '`' opening already consumed
   724  	offs := s.offset - 1
   725  
   726  	hasCR := false
   727  	for {
   728  		ch := s.ch
   729  		if ch < 0 {
   730  			s.error(offs, "raw string literal not terminated")
   731  			break
   732  		}
   733  		s.next()
   734  		if ch == '`' {
   735  			break
   736  		}
   737  		if ch == '\r' {
   738  			hasCR = true
   739  		}
   740  	}
   741  
   742  	lit := s.src[offs:s.offset]
   743  	if hasCR {
   744  		lit = stripCR(lit, false)
   745  	}
   746  
   747  	return string(lit)
   748  }
   749  
   750  func (s *Scanner) skipWhitespace() {
   751  	for s.ch == ' ' || s.ch == '\t' || s.ch == '\n' && !s.insertSemi || s.ch == '\r' {
   752  		s.next()
   753  	}
   754  }
   755  
   756  // Helper functions for scanning multi-byte tokens such as >> += >>= .
   757  // Different routines recognize different length tok_i based on matches
   758  // of ch_i. If a token ends in '=', the result is tok1 or tok3
   759  // respectively. Otherwise, the result is tok0 if there was no other
   760  // matching character, or tok2 if the matching character was ch2.
   761  
   762  func (s *Scanner) switch2(tok0, tok1 token.Token) token.Token {
   763  	if s.ch == '=' {
   764  		s.next()
   765  		return tok1
   766  	}
   767  	return tok0
   768  }
   769  
   770  func (s *Scanner) switch3(tok0, tok1 token.Token, ch2 rune, tok2 token.Token) token.Token {
   771  	if s.ch == '=' {
   772  		s.next()
   773  		return tok1
   774  	}
   775  	if s.ch == ch2 {
   776  		s.next()
   777  		return tok2
   778  	}
   779  	return tok0
   780  }
   781  
   782  func (s *Scanner) switch4(tok0, tok1 token.Token, ch2 rune, tok2, tok3 token.Token) token.Token {
   783  	if s.ch == '=' {
   784  		s.next()
   785  		return tok1
   786  	}
   787  	if s.ch == ch2 {
   788  		s.next()
   789  		if s.ch == '=' {
   790  			s.next()
   791  			return tok3
   792  		}
   793  		return tok2
   794  	}
   795  	return tok0
   796  }
   797  
   798  // Scan scans the next token and returns the token position, the token,
   799  // and its literal string if applicable. The source end is indicated by
   800  // token.EOF.
   801  //
   802  // If the returned token is a literal (token.IDENT, token.INT, token.FLOAT,
   803  // token.IMAG, token.CHAR, token.STRING) or token.COMMENT, the literal string
   804  // has the corresponding value.
   805  //
   806  // If the returned token is a keyword, the literal string is the keyword.
   807  //
   808  // If the returned token is token.SEMICOLON, the corresponding
   809  // literal string is ";" if the semicolon was present in the source,
   810  // and "\n" if the semicolon was inserted because of a newline or
   811  // at EOF.
   812  //
   813  // If the returned token is token.ILLEGAL, the literal string is the
   814  // offending character.
   815  //
   816  // In all other cases, Scan returns an empty literal string.
   817  //
   818  // For more tolerant parsing, Scan will return a valid token if
   819  // possible even if a syntax error was encountered. Thus, even
   820  // if the resulting token sequence contains no illegal tokens,
   821  // a client may not assume that no error occurred. Instead it
   822  // must check the scanner's ErrorCount or the number of calls
   823  // of the error handler, if there was one installed.
   824  //
   825  // Scan adds line information to the file added to the file
   826  // set with Init. Token positions are relative to that file
   827  // and thus relative to the file set.
   828  //
   829  func (s *Scanner) Scan() (pos token.Pos, tok token.Token, lit string) {
   830  scanAgain:
   831  	s.skipWhitespace()
   832  
   833  	// current token start
   834  	pos = s.file.Pos(s.offset)
   835  
   836  	// determine token value
   837  	insertSemi := false
   838  	switch ch := s.ch; {
   839  	case isLetter(ch):
   840  		lit = s.scanIdentifier()
   841  		if len(lit) > 1 {
   842  			// keywords are longer than one letter - avoid lookup otherwise
   843  			tok = token.Lookup(lit)
   844  			switch tok {
   845  			case token.IDENT, token.BREAK, token.CONTINUE, token.FALLTHROUGH, token.RETURN:
   846  				insertSemi = true
   847  			}
   848  		} else {
   849  			insertSemi = true
   850  			tok = token.IDENT
   851  		}
   852  	case isDecimal(ch) || ch == '.' && isDecimal(rune(s.peek())):
   853  		insertSemi = true
   854  		tok, lit = s.scanNumber()
   855  	default:
   856  		s.next() // always make progress
   857  		switch ch {
   858  		case -1:
   859  			if s.insertSemi {
   860  				s.insertSemi = false // EOF consumed
   861  				return pos, token.SEMICOLON, "\n"
   862  			}
   863  			tok = token.EOF
   864  		case '\n':
   865  			// we only reach here if s.insertSemi was
   866  			// set in the first place and exited early
   867  			// from s.skipWhitespace()
   868  			s.insertSemi = false // newline consumed
   869  			return pos, token.SEMICOLON, "\n"
   870  		case '"':
   871  			insertSemi = true
   872  			tok = token.STRING
   873  			lit = s.scanString()
   874  		case '\'':
   875  			insertSemi = true
   876  			tok = token.CHAR
   877  			lit = s.scanRune()
   878  		case '`':
   879  			insertSemi = true
   880  			tok = token.STRING
   881  			lit = s.scanRawString()
   882  		case ':':
   883  			tok = s.switch2(token.COLON, token.DEFINE)
   884  		case '.':
   885  			// fractions starting with a '.' are handled by outer switch
   886  			tok = token.PERIOD
   887  			if s.ch == '.' && s.peek() == '.' {
   888  				s.next()
   889  				s.next() // consume last '.'
   890  				tok = token.ELLIPSIS
   891  			}
   892  		case ',':
   893  			tok = token.COMMA
   894  		case ';':
   895  			tok = token.SEMICOLON
   896  			lit = ";"
   897  		case '(':
   898  			tok = token.LPAREN
   899  		case ')':
   900  			insertSemi = true
   901  			tok = token.RPAREN
   902  		case '[':
   903  			tok = token.LBRACK
   904  		case ']':
   905  			insertSemi = true
   906  			tok = token.RBRACK
   907  		case '{':
   908  			tok = token.LBRACE
   909  		case '}':
   910  			insertSemi = true
   911  			tok = token.RBRACE
   912  		case '+':
   913  			tok = s.switch3(token.ADD, token.ADD_ASSIGN, '+', token.INC)
   914  			if tok == token.INC {
   915  				insertSemi = true
   916  			}
   917  		case '-':
   918  			tok = s.switch3(token.SUB, token.SUB_ASSIGN, '-', token.DEC)
   919  			if tok == token.DEC {
   920  				insertSemi = true
   921  			}
   922  		case '*':
   923  			tok = s.switch2(token.MUL, token.MUL_ASSIGN)
   924  		case '/':
   925  			if s.ch == '/' || s.ch == '*' {
   926  				// comment
   927  				if s.insertSemi && s.findLineEnd() {
   928  					// reset position to the beginning of the comment
   929  					s.ch = '/'
   930  					s.offset = s.file.Offset(pos)
   931  					s.rdOffset = s.offset + 1
   932  					s.insertSemi = false // newline consumed
   933  					return pos, token.SEMICOLON, "\n"
   934  				}
   935  				comment := s.scanComment()
   936  				if s.mode&ScanComments == 0 {
   937  					// skip comment
   938  					s.insertSemi = false // newline consumed
   939  					goto scanAgain
   940  				}
   941  				tok = token.COMMENT
   942  				lit = comment
   943  			} else {
   944  				tok = s.switch2(token.QUO, token.QUO_ASSIGN)
   945  			}
   946  		case '%':
   947  			tok = s.switch2(token.REM, token.REM_ASSIGN)
   948  		case '^':
   949  			tok = s.switch2(token.XOR, token.XOR_ASSIGN)
   950  		case '<':
   951  			if s.ch == '-' {
   952  				s.next()
   953  				tok = token.ARROW
   954  			} else {
   955  				tok = s.switch4(token.LSS, token.LEQ, '<', token.SHL, token.SHL_ASSIGN)
   956  			}
   957  		case '>':
   958  			tok = s.switch4(token.GTR, token.GEQ, '>', token.SHR, token.SHR_ASSIGN)
   959  		case '=':
   960  			tok = s.switch2(token.ASSIGN, token.EQL)
   961  		case '!':
   962  			tok = s.switch2(token.NOT, token.NEQ)
   963  		case '&':
   964  			if s.ch == '^' {
   965  				s.next()
   966  				tok = s.switch2(token.AND_NOT, token.AND_NOT_ASSIGN)
   967  			} else {
   968  				tok = s.switch3(token.AND, token.AND_ASSIGN, '&', token.LAND)
   969  			}
   970  		case '|':
   971  			tok = s.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR)
   972  		default:
   973  			// next reports unexpected BOMs - don't repeat
   974  			if ch != bom {
   975  				s.errorf(s.file.Offset(pos), "illegal character %#U", ch)
   976  			}
   977  			insertSemi = s.insertSemi // preserve insertSemi info
   978  			tok = token.ILLEGAL
   979  			lit = string(ch)
   980  		}
   981  	}
   982  	if s.mode&dontInsertSemis == 0 {
   983  		s.insertSemi = insertSemi
   984  	}
   985  
   986  	return
   987  }
   988  

View as plain text