Lines Matching refs:tokenize
102 import tokenize
370 if (token_type == tokenize.OP and
373 prev_type == tokenize.NAME and
443 prev_type = tokenize.OP
446 if token_type in (tokenize.NL, tokenize.NEWLINE, tokenize.ERRORTOKEN):
457 elif token_type == tokenize.OP:
464 if ((prev_type != tokenize.OP or prev_text in '}])') and not
465 (prev_type == tokenize.NAME and iskeyword(prev_text))):
552 if token_type == tokenize.NL:
554 if token_type == tokenize.COMMENT:
765 Load a Python source file, tokenize it, check coding style.
794 used to feed tokenize.generate_tokens.
833 if token_type in (tokenize.COMMENT, tokenize.NL,
834 tokenize.INDENT, tokenize.DEDENT,
835 tokenize.NEWLINE):
837 if token_type == tokenize.STRING:
902 for token in tokenize.generate_tokens(self.readline_check_physical):
903 # print(tokenize.tok_name[token[0]], repr(token))
906 if token_type == tokenize.OP and text in '([{':
908 if token_type == tokenize.OP and text in '}])':
910 if token_type == tokenize.NEWLINE and not parens:
915 if token_type == tokenize.NL and not parens:
920 if token_type == tokenize.COMMENT:
