Lines Matching refs:tokens
68 # Token class. This class is used to represent the tokens produced.
134 self.lextokens = None # List of valid tokens
548 self.tokens = []
573 # Get the tokens map
575 tokens = self.ldict.get("tokens",None)
576 if not tokens:
581 if not isinstance(tokens,(list, tuple)):
582 self.log.error("tokens must be a list or tuple")
586 if not tokens:
587 self.log.error("tokens is empty")
591 self.tokens = tokens
593 # Validate the tokens
596 for n in self.tokens:
775 if not tokname in self.tokens and tokname.find("ignore_") < 0:
909 debuglog.info("lex: tokens = %r", linfo.tokens)
915 for n in linfo.tokens:
