Lines Matching refs:token
27 token = lexer.nextToken()
28 assert token.type == self.lexerModule.IDENTIFIER
29 assert token.start == 0, token.start
30 assert token.stop == 5, token.stop
31 assert token.text == 'foobar', token.text
33 token = lexer.nextToken()
34 assert token.type == self.lexerModule.WS
35 assert token.start == 6, token.start
36 assert token.stop == 6, token.stop
37 assert token.text == ' ', token.text
39 token = lexer.nextToken()
40 assert token.type == self.lexerModule.IDENTIFIER
41 assert token.start == 7, token.start
42 assert token.stop == 11, token.stop
43 assert token.text == '_Ab98', token.text
45 token = lexer.nextToken()
46 assert token.type == self.lexerModule.WS
47 assert token.start == 12, token.start
48 assert token.stop == 14, token.stop
49 assert token.text == ' \n ', token.text
51 token = lexer.nextToken()
52 assert token.type == self.lexerModule.IDENTIFIER
53 assert token.start == 15, token.start
54 assert token.stop == 20, token.stop
55 assert token.text == 'A12sdf', token.text
57 token = lexer.nextToken()
58 assert token.type == self.lexerModule.EOF
67 token = lexer.nextToken()
68 raise AssertionError, token
