Black Lives Matter. Support the Equal Justice Initiative.

Source file src/go/token/token.go

Documentation: go/token

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package token defines constants representing the lexical tokens of the Go
     6  // programming language and basic operations on tokens (printing, predicates).
     7  //
     8  package token
     9  
    10  import (
    11  	"strconv"
    12  	"unicode"
    13  	"unicode/utf8"
    14  )
    15  
    16  // Token is the set of lexical tokens of the Go programming language.
    17  type Token int
    18  
    19  // The list of tokens.
    20  const (
    21  	// Special tokens
    22  	ILLEGAL Token = iota
    23  	EOF
    24  	COMMENT
    25  
    26  	literal_beg
    27  	// Identifiers and basic type literals
    28  	// (these tokens stand for classes of literals)
    29  	IDENT  // main
    30  	INT    // 12345
    31  	FLOAT  // 123.45
    32  	IMAG   // 123.45i
    33  	CHAR   // 'a'
    34  	STRING // "abc"
    35  	literal_end
    36  
    37  	operator_beg
    38  	// Operators and delimiters
    39  	ADD // +
    40  	SUB // -
    41  	MUL // *
    42  	QUO // /
    43  	REM // %
    44  
    45  	AND     // &
    46  	OR      // |
    47  	XOR     // ^
    48  	SHL     // <<
    49  	SHR     // >>
    50  	AND_NOT // &^
    51  
    52  	ADD_ASSIGN // +=
    53  	SUB_ASSIGN // -=
    54  	MUL_ASSIGN // *=
    55  	QUO_ASSIGN // /=
    56  	REM_ASSIGN // %=
    57  
    58  	AND_ASSIGN     // &=
    59  	OR_ASSIGN      // |=
    60  	XOR_ASSIGN     // ^=
    61  	SHL_ASSIGN     // <<=
    62  	SHR_ASSIGN     // >>=
    63  	AND_NOT_ASSIGN // &^=
    64  
    65  	LAND  // &&
    66  	LOR   // ||
    67  	ARROW // <-
    68  	INC   // ++
    69  	DEC   // --
    70  
    71  	EQL    // ==
    72  	LSS    // <
    73  	GTR    // >
    74  	ASSIGN // =
    75  	NOT    // !
    76  
    77  	NEQ      // !=
    78  	LEQ      // <=
    79  	GEQ      // >=
    80  	DEFINE   // :=
    81  	ELLIPSIS // ...
    82  
    83  	LPAREN // (
    84  	LBRACK // [
    85  	LBRACE // {
    86  	COMMA  // ,
    87  	PERIOD // .
    88  
    89  	RPAREN    // )
    90  	RBRACK    // ]
    91  	RBRACE    // }
    92  	SEMICOLON // ;
    93  	COLON     // :
    94  	operator_end
    95  
    96  	keyword_beg
    97  	// Keywords
    98  	BREAK
    99  	CASE
   100  	CHAN
   101  	CONST
   102  	CONTINUE
   103  
   104  	DEFAULT
   105  	DEFER
   106  	ELSE
   107  	FALLTHROUGH
   108  	FOR
   109  
   110  	FUNC
   111  	GO
   112  	GOTO
   113  	IF
   114  	IMPORT
   115  
   116  	INTERFACE
   117  	MAP
   118  	PACKAGE
   119  	RANGE
   120  	RETURN
   121  
   122  	SELECT
   123  	STRUCT
   124  	SWITCH
   125  	TYPE
   126  	VAR
   127  	keyword_end
   128  )
   129  
   130  var tokens = [...]string{
   131  	ILLEGAL: "ILLEGAL",
   132  
   133  	EOF:     "EOF",
   134  	COMMENT: "COMMENT",
   135  
   136  	IDENT:  "IDENT",
   137  	INT:    "INT",
   138  	FLOAT:  "FLOAT",
   139  	IMAG:   "IMAG",
   140  	CHAR:   "CHAR",
   141  	STRING: "STRING",
   142  
   143  	ADD: "+",
   144  	SUB: "-",
   145  	MUL: "*",
   146  	QUO: "/",
   147  	REM: "%",
   148  
   149  	AND:     "&",
   150  	OR:      "|",
   151  	XOR:     "^",
   152  	SHL:     "<<",
   153  	SHR:     ">>",
   154  	AND_NOT: "&^",
   155  
   156  	ADD_ASSIGN: "+=",
   157  	SUB_ASSIGN: "-=",
   158  	MUL_ASSIGN: "*=",
   159  	QUO_ASSIGN: "/=",
   160  	REM_ASSIGN: "%=",
   161  
   162  	AND_ASSIGN:     "&=",
   163  	OR_ASSIGN:      "|=",
   164  	XOR_ASSIGN:     "^=",
   165  	SHL_ASSIGN:     "<<=",
   166  	SHR_ASSIGN:     ">>=",
   167  	AND_NOT_ASSIGN: "&^=",
   168  
   169  	LAND:  "&&",
   170  	LOR:   "||",
   171  	ARROW: "<-",
   172  	INC:   "++",
   173  	DEC:   "--",
   174  
   175  	EQL:    "==",
   176  	LSS:    "<",
   177  	GTR:    ">",
   178  	ASSIGN: "=",
   179  	NOT:    "!",
   180  
   181  	NEQ:      "!=",
   182  	LEQ:      "<=",
   183  	GEQ:      ">=",
   184  	DEFINE:   ":=",
   185  	ELLIPSIS: "...",
   186  
   187  	LPAREN: "(",
   188  	LBRACK: "[",
   189  	LBRACE: "{",
   190  	COMMA:  ",",
   191  	PERIOD: ".",
   192  
   193  	RPAREN:    ")",
   194  	RBRACK:    "]",
   195  	RBRACE:    "}",
   196  	SEMICOLON: ";",
   197  	COLON:     ":",
   198  
   199  	BREAK:    "break",
   200  	CASE:     "case",
   201  	CHAN:     "chan",
   202  	CONST:    "const",
   203  	CONTINUE: "continue",
   204  
   205  	DEFAULT:     "default",
   206  	DEFER:       "defer",
   207  	ELSE:        "else",
   208  	FALLTHROUGH: "fallthrough",
   209  	FOR:         "for",
   210  
   211  	FUNC:   "func",
   212  	GO:     "go",
   213  	GOTO:   "goto",
   214  	IF:     "if",
   215  	IMPORT: "import",
   216  
   217  	INTERFACE: "interface",
   218  	MAP:       "map",
   219  	PACKAGE:   "package",
   220  	RANGE:     "range",
   221  	RETURN:    "return",
   222  
   223  	SELECT: "select",
   224  	STRUCT: "struct",
   225  	SWITCH: "switch",
   226  	TYPE:   "type",
   227  	VAR:    "var",
   228  }
   229  
   230  // String returns the string corresponding to the token tok.
   231  // For operators, delimiters, and keywords the string is the actual
   232  // token character sequence (e.g., for the token ADD, the string is
   233  // "+"). For all other tokens the string corresponds to the token
   234  // constant name (e.g. for the token IDENT, the string is "IDENT").
   235  //
   236  func (tok Token) String() string {
   237  	s := ""
   238  	if 0 <= tok && tok < Token(len(tokens)) {
   239  		s = tokens[tok]
   240  	}
   241  	if s == "" {
   242  		s = "token(" + strconv.Itoa(int(tok)) + ")"
   243  	}
   244  	return s
   245  }
   246  
   247  // A set of constants for precedence-based expression parsing.
   248  // Non-operators have lowest precedence, followed by operators
   249  // starting with precedence 1 up to unary operators. The highest
   250  // precedence serves as "catch-all" precedence for selector,
   251  // indexing, and other operator and delimiter tokens.
   252  //
   253  const (
   254  	LowestPrec  = 0 // non-operators
   255  	UnaryPrec   = 6
   256  	HighestPrec = 7
   257  )
   258  
   259  // Precedence returns the operator precedence of the binary
   260  // operator op. If op is not a binary operator, the result
   261  // is LowestPrecedence.
   262  //
   263  func (op Token) Precedence() int {
   264  	switch op {
   265  	case LOR:
   266  		return 1
   267  	case LAND:
   268  		return 2
   269  	case EQL, NEQ, LSS, LEQ, GTR, GEQ:
   270  		return 3
   271  	case ADD, SUB, OR, XOR:
   272  		return 4
   273  	case MUL, QUO, REM, SHL, SHR, AND, AND_NOT:
   274  		return 5
   275  	}
   276  	return LowestPrec
   277  }
   278  
   279  var keywords map[string]Token
   280  
   281  func init() {
   282  	keywords = make(map[string]Token)
   283  	for i := keyword_beg + 1; i < keyword_end; i++ {
   284  		keywords[tokens[i]] = i
   285  	}
   286  }
   287  
   288  // Lookup maps an identifier to its keyword token or IDENT (if not a keyword).
   289  //
   290  func Lookup(ident string) Token {
   291  	if tok, is_keyword := keywords[ident]; is_keyword {
   292  		return tok
   293  	}
   294  	return IDENT
   295  }
   296  
   297  // Predicates
   298  
   299  // IsLiteral returns true for tokens corresponding to identifiers
   300  // and basic type literals; it returns false otherwise.
   301  //
   302  func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_end }
   303  
   304  // IsOperator returns true for tokens corresponding to operators and
   305  // delimiters; it returns false otherwise.
   306  //
   307  func (tok Token) IsOperator() bool { return operator_beg < tok && tok < operator_end }
   308  
   309  // IsKeyword returns true for tokens corresponding to keywords;
   310  // it returns false otherwise.
   311  //
   312  func (tok Token) IsKeyword() bool { return keyword_beg < tok && tok < keyword_end }
   313  
   314  // IsExported reports whether name starts with an upper-case letter.
   315  //
   316  func IsExported(name string) bool {
   317  	ch, _ := utf8.DecodeRuneInString(name)
   318  	return unicode.IsUpper(ch)
   319  }
   320  
   321  // IsKeyword reports whether name is a Go keyword, such as "func" or "return".
   322  //
   323  func IsKeyword(name string) bool {
   324  	// TODO: opt: use a perfect hash function instead of a global map.
   325  	_, ok := keywords[name]
   326  	return ok
   327  }
   328  
   329  // IsIdentifier reports whether name is a Go identifier, that is, a non-empty
   330  // string made up of letters, digits, and underscores, where the first character
   331  // is not a digit. Keywords are not identifiers.
   332  //
   333  func IsIdentifier(name string) bool {
   334  	for i, c := range name {
   335  		if !unicode.IsLetter(c) && c != '_' && (i == 0 || !unicode.IsDigit(c)) {
   336  			return false
   337  		}
   338  	}
   339  	return name != "" && !IsKeyword(name)
   340  }
   341  

View as plain text