Lines Matching full:token
27 token = lexer.nextToken()
28 assert token.type == self.lexerModule.FOO
29 assert token.start == 0, token.start
30 assert token.stop == 1, token.stop
31 assert token.text == 'fo', token.text
33 token = lexer.nextToken()
34 assert token.type == self.lexerModule.FOO
35 assert token.start == 2, token.start
36 assert token.stop == 4, token.stop
37 assert token.text == 'foo', token.text
39 token = lexer.nextToken()
40 assert token.type == self.lexerModule.FOO
41 assert token.start == 5, token.start
42 assert token.stop == 8, token.stop
43 assert token.text == 'fooo', token.text
45 token = lexer.nextToken()
46 assert token.type == self.lexerModule.EOF
54 token = lexer.nextToken()
67 token = lexer.nextToken()
