1 #! /usr/bin/env python 2 3 """Token constants (from "token.h").""" 4 5 # Taken from Python (r53757) and modified to include some tokens 6 # originally monkeypatched in by pgen2.tokenize 7 8 #--start constants-- 9 ENDMARKER = 0 10 NAME = 1 11 NUMBER = 2 12 STRING = 3 13 NEWLINE = 4 14 INDENT = 5 15 DEDENT = 6 16 LPAR = 7 17 RPAR = 8 18 LSQB = 9 19 RSQB = 10 20 COLON = 11 21 COMMA = 12 22 SEMI = 13 23 PLUS = 14 24 MINUS = 15 25 STAR = 16 26 SLASH = 17 27 VBAR = 18 28 AMPER = 19 29 LESS = 20 30 GREATER = 21 31 EQUAL = 22 32 DOT = 23 33 PERCENT = 24 34 BACKQUOTE = 25 35 LBRACE = 26 36 RBRACE = 27 37 EQEQUAL = 28 38 NOTEQUAL = 29 39 LESSEQUAL = 30 40 GREATEREQUAL = 31 41 TILDE = 32 42 CIRCUMFLEX = 33 43 LEFTSHIFT = 34 44 RIGHTSHIFT = 35 45 DOUBLESTAR = 36 46 PLUSEQUAL = 37 47 MINEQUAL = 38 48 STAREQUAL = 39 49 SLASHEQUAL = 40 50 PERCENTEQUAL = 41 51 AMPEREQUAL = 42 52 VBAREQUAL = 43 53 CIRCUMFLEXEQUAL = 44 54 LEFTSHIFTEQUAL = 45 55 RIGHTSHIFTEQUAL = 46 56 DOUBLESTAREQUAL = 47 57 DOUBLESLASH = 48 58 DOUBLESLASHEQUAL = 49 59 AT = 50 60 ATEQUAL = 51 61 OP = 52 62 COMMENT = 53 63 NL = 54 64 RARROW = 55 65 ERRORTOKEN = 56 66 N_TOKENS = 57 67 NT_OFFSET = 256 68 #--end constants-- 69 70 tok_name = {} 71 for _name, _value in globals().items(): 72 if type(_value) is type(0): 73 tok_name[_value] = _name 74 75 76 def ISTERMINAL(x): 77 return x < NT_OFFSET 78 79 def ISNONTERMINAL(x): 80 return x >= NT_OFFSET 81 82 def ISEOF(x): 83 return x == ENDMARKER 84
