Lines Matching refs:tokens
383 """A generator that yields the tokens in the given string."""
475 def ParseElseNode(tokens):
477 return PopToken(tokens, token_type)
479 next = PeekToken(tokens)
485 code_node = ParseCodeNode(tokens)
492 code_node = ParseCodeNode(tokens)
494 inner_else_node = ParseElseNode(tokens)
498 return ParseElseNode(tokens)
503 def ParseAtomicCodeNode(tokens):
505 return PopToken(tokens, token_type)
507 head = PopFront(tokens)
514 next = PeekToken(tokens)
519 code_node = ParseCodeNode(tokens)
524 next_token = PeekToken(tokens)
531 code_node = ParseCodeNode(tokens)
537 code_node = ParseCodeNode(tokens)
539 else_node = ParseElseNode(tokens)
556 code_node = ParseCodeNode(tokens)
560 PushFront(tokens, head)
564 def ParseCodeNode(tokens):
567 if not tokens:
569 atomic_code_node = ParseAtomicCodeNode(tokens)
579 tokens = list(Tokenize(pump_src_text))
580 code_node = ParseCodeNode(tokens)
