Lines Matching full:line
5 repeatedly to get the next line of input (or "" for EOF). It generates
12 the original line (string)
74 # Single-line ' or " string.
93 # First (or only) line of ' or " string.
149 def printtoken(type, token, srow_scol, erow_ecol, line): # for testing
162 Each call to the function should return one line of input as a string.
197 tok_type, token, start, end, line = t
269 should return one line of input as a string. Alternately, readline
277 and the line on which the token was found. The line passed is the
278 logical line; continuation lines are included.
288 line = readline()
290 line = ''
292 pos, max = 0, len(line)
295 if not line:
296 raise TokenError, ("EOF in multi-line string", strstart)
297 endmatch = endprog.match(line)
300 yield (STRING, contstr + line[:end],
301 strstart, (lnum, end), contline + line)
304 elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
305 yield (ERRORTOKEN, contstr + line,
306 strstart, (lnum, len(line)), contline)
311 contstr = contstr + line
312 contline = contline + line
316 if not line: break
319 if line[pos] == ' ':
321 elif line[pos] == '\t':
323 elif line[pos] == '\f':
331 if line[pos] in '#\r\n': # skip comments or blank lines
332 if line[pos] == '#':
333 comment_token = line[pos:].rstrip('\r\n')
336 (lnum, pos), (lnum, pos + len(comment_token)), line)
337 yield (NL, line[nl_pos:],
338 (lnum, nl_pos), (lnum, len(line)), line)
340 yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],
341 (lnum, pos), (lnum, len(line)), line)
346 yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
351 ("<tokenize>", lnum, pos, line))
353 yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
356 if not line:
357 raise TokenError, ("EOF in multi-line statement", (lnum, 0))
361 pseudomatch = pseudoprog.match(line, pos)
367 token, initial = line[start:end], line[start]
371 yield (NUMBER, token, spos, epos, line)
374 token, spos, epos, line)
377 yield (COMMENT, token, spos, epos, line)
380 endmatch = endprog.match(line, pos)
381 if endmatch: # all on one line
383 token = line[start:pos]
384 yield (STRING, token, spos, (lnum, pos), line)
387 contstr = line[start:]
388 contline = line
397 contstr, needcont = line[start:], 1
398 contline = line
401 yield (STRING, token, spos, epos, line)
403 yield (NAME, token, spos, epos, line)
411 yield (OP, token, spos, epos, line)
413 yield (ERRORTOKEN, line[pos],
414 (lnum, pos), (lnum, pos+1), line)
