Lines Matching defs:TOKEN
57 # This regular expression is used to match valid token names
60 # Exception thrown when invalid token encountered and no default error
68 # Token class. This class is used to represent the tokens produced.
107 # token() - Get the next token
298 # opttoken() - Return the next token from the Lexer
304 def token(self):
322 # Create a token for return
332 # If no token type was set, it's an ignored token
342 # If token is processed by a function, call it
344 tok.lexer = self # Set additional attributes useful in token rules
350 # Every function must return a token, if nothing, we just move to next token
356 # Verify type of the token. If not in the token map, raise an error
359 raise LexError("%s:%d: Rule '%s' returned an unknown token type '%s'" % (
405 t = self.token()
517 # is a tuple of state names and tokenname is the name of the token. For example,
577 self.log.error("No token list is defined")
598 self.log.error("Bad token name '%s'",n)
601 self.log.warning("Token '%s' multiply defined", n)
657 self.toknames = { } # Mapping of symbols to token names
776 self.log.error("Rule '%s' defined for an unspecified token %s",name,tokname)
871 global token,input
899 token = lexobj.token
913 # Build a dictionary of valid token names
997 # Create global versions of the token() and input() functions
998 token = lexobj.token
1031 _token = lexer.token
1033 _token = token
1041 # @TOKEN(regex)
1047 def TOKEN(r):
1056 # Alternative spelling of the TOKEN decorator
1057 Token = TOKEN
