1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Package token defines constants representing the lexical tokens of the Go 6 // programming language and basic operations on tokens (printing, predicates). 7 // 8 package token 9 10 import "strconv" 11 12 // Token is the set of lexical tokens of the Go programming language. 13 type Token int 14 15 // The list of tokens. 16 const ( 17 // Special tokens 18 ILLEGAL Token = iota 19 EOF 20 COMMENT 21 22 literal_beg 23 // Identifiers and basic type literals 24 // (these tokens stand for classes of literals) 25 IDENT // main 26 INT // 12345 27 FLOAT // 123.45 28 IMAG // 123.45i 29 CHAR // 'a' 30 STRING // "abc" 31 literal_end 32 33 operator_beg 34 // Operators and delimiters 35 ADD // + 36 SUB // - 37 MUL // * 38 QUO // / 39 REM // % 40 41 AND // & 42 OR // | 43 XOR // ^ 44 SHL // << 45 SHR // >> 46 AND_NOT // &^ 47 48 ADD_ASSIGN // += 49 SUB_ASSIGN // -= 50 MUL_ASSIGN // *= 51 QUO_ASSIGN // /= 52 REM_ASSIGN // %= 53 54 AND_ASSIGN // &= 55 OR_ASSIGN // |= 56 XOR_ASSIGN // ^= 57 SHL_ASSIGN // <<= 58 SHR_ASSIGN // >>= 59 AND_NOT_ASSIGN // &^= 60 61 LAND // && 62 LOR // || 63 ARROW // <- 64 INC // ++ 65 DEC // -- 66 67 EQL // == 68 LSS // < 69 GTR // > 70 ASSIGN // = 71 NOT // ! 72 73 NEQ // != 74 LEQ // <= 75 GEQ // >= 76 DEFINE // := 77 ELLIPSIS // ... 78 79 LPAREN // ( 80 LBRACK // [ 81 LBRACE // { 82 COMMA // , 83 PERIOD // . 84 85 RPAREN // ) 86 RBRACK // ] 87 RBRACE // } 88 SEMICOLON // ; 89 COLON // : 90 operator_end 91 92 keyword_beg 93 // Keywords 94 BREAK 95 CASE 96 CHAN 97 CONST 98 CONTINUE 99 100 DEFAULT 101 DEFER 102 ELSE 103 FALLTHROUGH 104 FOR 105 106 FUNC 107 GO 108 GOTO 109 IF 110 IMPORT 111 112 INTERFACE 113 MAP 114 PACKAGE 115 RANGE 116 RETURN 117 118 SELECT 119 STRUCT 120 SWITCH 121 TYPE 122 VAR 123 keyword_end 124 ) 125 126 var tokens = [...]string{ 127 ILLEGAL: "ILLEGAL", 128 129 EOF: "EOF", 130 COMMENT: "COMMENT", 131 132 IDENT: "IDENT", 133 INT: "INT", 134 FLOAT: "FLOAT", 135 IMAG: "IMAG", 136 CHAR: "CHAR", 137 STRING: "STRING", 138 139 ADD: "+", 140 SUB: "-", 141 MUL: "*", 142 QUO: "/", 143 REM: "%", 144 145 AND: "&", 146 OR: "|", 147 XOR: "^", 148 SHL: "<<", 149 SHR: ">>", 150 AND_NOT: "&^", 151 152 ADD_ASSIGN: "+=", 153 SUB_ASSIGN: "-=", 154 MUL_ASSIGN: "*=", 155 QUO_ASSIGN: "/=", 156 REM_ASSIGN: "%=", 157 158 AND_ASSIGN: "&=", 159 OR_ASSIGN: "|=", 160 XOR_ASSIGN: "^=", 161 SHL_ASSIGN: "<<=", 162 SHR_ASSIGN: ">>=", 163 AND_NOT_ASSIGN: "&^=", 164 165 LAND: "&&", 166 LOR: "||", 167 ARROW: "<-", 168 INC: "++", 169 DEC: "--", 170 171 EQL: "==", 172 LSS: "<", 173 GTR: ">", 174 ASSIGN: "=", 175 NOT: "!", 176 177 NEQ: "!=", 178 LEQ: "<=", 179 GEQ: ">=", 180 DEFINE: ":=", 181 ELLIPSIS: "...", 182 183 LPAREN: "(", 184 LBRACK: "[", 185 LBRACE: "{", 186 COMMA: ",", 187 PERIOD: ".", 188 189 RPAREN: ")", 190 RBRACK: "]", 191 RBRACE: "}", 192 SEMICOLON: ";", 193 COLON: ":", 194 195 BREAK: "break", 196 CASE: "case", 197 CHAN: "chan", 198 CONST: "const", 199 CONTINUE: "continue", 200 201 DEFAULT: "default", 202 DEFER: "defer", 203 ELSE: "else", 204 FALLTHROUGH: "fallthrough", 205 FOR: "for", 206 207 FUNC: "func", 208 GO: "go", 209 GOTO: "goto", 210 IF: "if", 211 IMPORT: "import", 212 213 INTERFACE: "interface", 214 MAP: "map", 215 PACKAGE: "package", 216 RANGE: "range", 217 RETURN: "return", 218 219 SELECT: "select", 220 STRUCT: "struct", 221 SWITCH: "switch", 222 TYPE: "type", 223 VAR: "var", 224 } 225 226 // String returns the string corresponding to the token tok. 227 // For operators, delimiters, and keywords the string is the actual 228 // token character sequence (e.g., for the token ADD, the string is 229 // "+"). For all other tokens the string corresponds to the token 230 // constant name (e.g. for the token IDENT, the string is "IDENT"). 231 // 232 func (tok Token) String() string { 233 s := "" 234 if 0 <= tok && tok < Token(len(tokens)) { 235 s = tokens[tok] 236 } 237 if s == "" { 238 s = "token(" + strconv.Itoa(int(tok)) + ")" 239 } 240 return s 241 } 242 243 // A set of constants for precedence-based expression parsing. 244 // Non-operators have lowest precedence, followed by operators 245 // starting with precedence 1 up to unary operators. The highest 246 // precedence serves as "catch-all" precedence for selector, 247 // indexing, and other operator and delimiter tokens. 248 // 249 const ( 250 LowestPrec = 0 // non-operators 251 UnaryPrec = 6 252 HighestPrec = 7 253 ) 254 255 // Precedence returns the operator precedence of the binary 256 // operator op. If op is not a binary operator, the result 257 // is LowestPrecedence. 258 // 259 func (op Token) Precedence() int { 260 switch op { 261 case LOR: 262 return 1 263 case LAND: 264 return 2 265 case EQL, NEQ, LSS, LEQ, GTR, GEQ: 266 return 3 267 case ADD, SUB, OR, XOR: 268 return 4 269 case MUL, QUO, REM, SHL, SHR, AND, AND_NOT: 270 return 5 271 } 272 return LowestPrec 273 } 274 275 var keywords map[string]Token 276 277 func init() { 278 keywords = make(map[string]Token) 279 for i := keyword_beg + 1; i < keyword_end; i++ { 280 keywords[tokens[i]] = i 281 } 282 } 283 284 // Lookup maps an identifier to its keyword token or IDENT (if not a keyword). 285 // 286 func Lookup(ident string) Token { 287 if tok, is_keyword := keywords[ident]; is_keyword { 288 return tok 289 } 290 return IDENT 291 } 292 293 // Predicates 294 295 // IsLiteral returns true for tokens corresponding to identifiers 296 // and basic type literals; it returns false otherwise. 297 // 298 func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_end } 299 300 // IsOperator returns true for tokens corresponding to operators and 301 // delimiters; it returns false otherwise. 302 // 303 func (tok Token) IsOperator() bool { return operator_beg < tok && tok < operator_end } 304 305 // IsKeyword returns true for tokens corresponding to keywords; 306 // it returns false otherwise. 307 // 308 func (tok Token) IsKeyword() bool { return keyword_beg < tok && tok < keyword_end } 309
