Lines Matching refs:tokenize
42 import tokenize
145 # Line-getter for tokenize.
157 STRING = tokenize.STRING
158 NL = tokenize.NL
159 NEWLINE = tokenize.NEWLINE
160 COMMENT = tokenize.COMMENT
161 NAME = tokenize.NAME
162 OP = tokenize.OP
165 get = tokenize.generate_tokens(self.getline).next
184 startline = srow - 1 # tokenize is one-based
