Home | History | Annotate | Download | only in pgen2
      1 #! /usr/bin/env python
      2 
      3 """Token constants (from "token.h")."""
      4 
      5 #  Taken from Python (r53757) and modified to include some tokens
      6 #   originally monkeypatched in by pgen2.tokenize
      7 
      8 #--start constants--
      9 ENDMARKER = 0
     10 NAME = 1
     11 NUMBER = 2
     12 STRING = 3
     13 NEWLINE = 4
     14 INDENT = 5
     15 DEDENT = 6
     16 LPAR = 7
     17 RPAR = 8
     18 LSQB = 9
     19 RSQB = 10
     20 COLON = 11
     21 COMMA = 12
     22 SEMI = 13
     23 PLUS = 14
     24 MINUS = 15
     25 STAR = 16
     26 SLASH = 17
     27 VBAR = 18
     28 AMPER = 19
     29 LESS = 20
     30 GREATER = 21
     31 EQUAL = 22
     32 DOT = 23
     33 PERCENT = 24
     34 BACKQUOTE = 25
     35 LBRACE = 26
     36 RBRACE = 27
     37 EQEQUAL = 28
     38 NOTEQUAL = 29
     39 LESSEQUAL = 30
     40 GREATEREQUAL = 31
     41 TILDE = 32
     42 CIRCUMFLEX = 33
     43 LEFTSHIFT = 34
     44 RIGHTSHIFT = 35
     45 DOUBLESTAR = 36
     46 PLUSEQUAL = 37
     47 MINEQUAL = 38
     48 STAREQUAL = 39
     49 SLASHEQUAL = 40
     50 PERCENTEQUAL = 41
     51 AMPEREQUAL = 42
     52 VBAREQUAL = 43
     53 CIRCUMFLEXEQUAL = 44
     54 LEFTSHIFTEQUAL = 45
     55 RIGHTSHIFTEQUAL = 46
     56 DOUBLESTAREQUAL = 47
     57 DOUBLESLASH = 48
     58 DOUBLESLASHEQUAL = 49
     59 AT = 50
     60 ATEQUAL = 51
     61 OP = 52
     62 COMMENT = 53
     63 NL = 54
     64 RARROW = 55
     65 ERRORTOKEN = 56
     66 N_TOKENS = 57
     67 NT_OFFSET = 256
     68 #--end constants--
     69 
     70 tok_name = {}
     71 for _name, _value in globals().items():
     72     if type(_value) is type(0):
     73         tok_name[_value] = _name
     74 
     75 
     76 def ISTERMINAL(x):
     77     return x < NT_OFFSET
     78 
     79 def ISNONTERMINAL(x):
     80     return x >= NT_OFFSET
     81 
     82 def ISEOF(x):
     83     return x == ENDMARKER
     84