Lines Matching refs:tokens
110 p = subprocess.Popen(['clang','-dump-raw-tokens',path],
116 tokens = []
127 tokens.append(Token(*kTokenRE.match(ln).groups()))
129 return tokens
154 for i,(file,tokens) in enumerate(self.tokenLists):
157 f.write(tokens[j])
174 sys.stderr.write('%s:%d tokens: [' % (file,len(byFile[i])))
187 print >>sys.stderr, ', '.join(['%s:%d tokens' % (file, len(byFile[i]))
200 print '\nSUCCESS (%d tokens)' % len(changes)
208 for i,(file,tokens) in enumerate(self.tokenLists)
209 for j in range(len(tokens))])
216 # Read in the lists of tokens.
220 numTokens = sum([len(tokens) for _,tokens in tokenLists])
221 print "Delta on %s with %d tokens." % (', '.join(files), numTokens)
227 print "Finished %s with %d tokens (in %d tests)." % (', '.join(tbmd.targetFiles),
