Lines Matching refs:token
11 the token type (see token.py)
12 the token (a string)
13 the starting (row, column) indices of the token (a 2-tuple of ints)
14 the ending (row, column) indices of the token (a 2-tuple of ints)
26 each time a new token is found."""
34 from lib2to3.pgen2.token import *
36 from . import token
37 __all__ = [x for x in dir(token) if x[0] != '_'] + ["tokenize",
39 del token
95 Token = Ignore + PlainToken
106 re.compile, (Token, PseudoToken, Single3, Double3))
154 def printtoken(type, token, start, end, line): # for testing
158 (srow, scol, erow, ecol, tok_name[type], repr(token))
170 called once for each token, with five arguments, corresponding to the
202 tok_type, token, start, end, line = t
204 self.tokens.append(token)
211 def compat(self, token, iterable):
215 toknum, tokval = token
327 Each element returned by the iterable must be a token sequence
328 with at least two elements, a token number and token value. If
354 The generator produces 5-tuples with these members: the token type; the
355 token string; a 2-tuple (srow, scol) of ints specifying the row and
356 column where the token begins in the source; a 2-tuple (erow, ecol) of
357 ints specifying the row and column where the token ends in the source;
358 and the line on which the token was found. The line passed is the
441 token, initial = line[start:end], line[start]
444 (initial == '.' and token != '.'): # ordinary number
445 yield (NUMBER, token, spos, epos, line)
450 yield (newline, token, spos, epos, line)
452 assert not token.endswith("\n")
453 yield (COMMENT, token, spos, epos, line)
454 elif token in triple_quoted:
455 endprog = endprogs[token]
459 token = line[start:pos]
460 yield (STRING, token, spos, (lnum, pos), line)
467 token[:2] in single_quoted or \
468 token[:3] in single_quoted:
469 if token[-1] == '\n': # continued string
471 endprog = (endprogs[initial] or endprogs[token[1]] or
472 endprogs[token[2]])
477 yield (STRING, token, spos, epos, line)
479 yield (NAME, token, spos, epos, line)
482 yield (NL, token, spos, (lnum, pos), line)
487 yield (OP, token, spos, epos, line)
