Lines Matching refs:token
20 token = lexer.next_token
21 token.name.should == 'ZERO'
23 token = lexer.next_token
24 token.name.should == '<EOF>'
30 token_types = lexer.map { |token| token.name }
34 example "mismatched token" do
38 token = lexer.next_token
62 token = lexer.next_token
63 token.name.should == 'ZERO'
65 token = lexer.next_token
66 token.name.should == 'ONE'
68 token = lexer.next_token
69 token.name.should == '<EOF>'
72 example "no matching token rule" do
75 b = lambda { token = lexer.next_token }
100 token = lexer.next_token
101 token.name.should == 'ZERO'
103 token = lexer.next_token
104 token.name.should == 'FOOZE'
106 token = lexer.next_token
107 token.name.should == 'ONE'
109 token = lexer.next_token
110 token.name.should == '<EOF>'
113 example "no token match" do
139 token = lexer.next_token
140 token.name.should == 'FOO'
141 token.start.should == 0
142 token.stop.should == 0
143 token.text.should == 'f'
145 token = lexer.next_token
146 token.name.should == 'FOO'
147 token.text.should == 'fo'
148 token.start.should == 1
149 token.stop.should == 2
151 token = lexer.next_token
152 token.name.should == 'FOO'
153 token.start.should == 3
154 token.stop.should == 5
155 token.text.should == 'foo'
157 token = lexer.next_token
158 token.name.should == 'FOO'
159 token.start.should == 6
160 token.stop.should == 9
161 token.text.should == 'fooo'
163 token = lexer.next_token
164 token.name.should == '<EOF>'
167 example "mismatched token" do
193 token = lexer.next_token
194 token.name.should == 'FOO'
195 token.start.should == 0
196 token.stop.should == 1
197 token.text.should == 'fo'
199 token = lexer.next_token
200 token.name.should == 'FOO'
201 token.text.should == 'foo'
202 token.start.should == 2
203 token.stop.should == 4
205 token = lexer.next_token
206 token.name.should == 'FOO'
207 token.start.should == 5
208 token.stop.should == 8
209 token.text.should == 'fooo'
211 token = lexer.next_token
212 token.name.should == '<EOF>'
215 example "mismatched token" do
228 proc { token = lexer.next_token }.
251 token = lexer.next_token
252 token.name.should == 'FOO'
253 token.start.should == 0
254 token.stop.should == 1
255 token.text.should == 'fo'
257 token = lexer.next_token
258 token.name.should == 'FOO'
259 token.text.should == 'faaooa'
260 token.start.should == 2
261 token.stop.should == 7
263 token = lexer.next_token
264 token.name.should == '<EOF>'
267 example "mismatched token" do
297 token = lexer.next_token
298 token.name.should == 'FOO'
299 token.start.should == 0
300 token.stop.should == 1
301 token.text.should == 'fo'
303 token = lexer.next_token
304 token.name.should == 'FOO'
305 token.start.should == 2
306 token.stop.should == 12
307 token.text.should == 'fababbooabb'
309 token = lexer.next_token
310 token.name.should == '<EOF>'
340 token = lexer.next_token
341 token.name.should == 'FOO'
342 token.start.should == 0
343 token.stop.should == 0
344 token.text.should == 'f'
346 token = lexer.next_token
347 token.name.should == 'FOO'
348 token.start.should == 1
349 token.stop.should == 2
350 token.text.should == 'fa'
352 token = lexer.next_token
353 token.name.should == 'FOO'
354 token.start.should == 3
355 token.stop.should == 3
356 token.text.should == 'f'
358 token = lexer.next_token
359 token.name.should == '<EOF>'
362 example "mismatched token" do
392 token = lexer.next_token
393 token.name.should == 'DIGIT'
394 token.start.should == 0
395 token.stop.should == 0
396 token.text.should == '0'
398 token = lexer.next_token
399 token.name.should == 'DIGIT'
400 token.start.should == 1
401 token.stop.should == 1
402 token.text.should == '8'
404 token = lexer.next_token
405 token.name.should == 'DIGIT'
406 token.start.should == 2
407 token.stop.should == 2
408 token.text.should == '5'
410 token = lexer.next_token
411 token.name.should == '<EOF>'
445 token = lexer.next_token
446 token.name.should == 'IDENTIFIER'
447 token.start.should == 0
448 token.stop.should == 5
449 token.text.should == 'foobar'
451 token = lexer.next_token
452 token.name.should == 'WS'
453 token.start.should == 6
454 token.stop.should == 6
455 token.text.should == ' '
457 token = lexer.next_token
458 token.name.should == 'IDENTIFIER'
459 token.start.should == 7
460 token.stop.should == 11
461 token.text.should == '_Ab98'
463 token = lexer.next_token
464 token.name.should == 'WS'
465 token.start.should == 12
466 token.stop.should == 14
467 token.text.should == " \n "
469 token = lexer.next_token
470 token.name.should == 'IDENTIFIER'
471 token.start.should == 15
472 token.stop.should == 20
473 token.text.should == 'A12sdf'
475 token = lexer.next_token
476 token.name.should == '<EOF>'
479 example "contains characters without a matching token rule" do
514 token = lexer.next_token
515 token.name.should == 'IDENTIFIER'
516 token.start.should == 0
517 token.stop.should == 5
518 token.text.should == 'foobar'
520 token = lexer.next_token
521 token.name.should == 'WS'
522 token.start.should == 6
523 token.stop.should == 6
524 token.text.should == ' '
526 token = lexer.next_token
527 token.name.should == 'IDENTIFIER'
528 token.start.should == 7
529 token.stop.should == 11
530 token.text.should == '_Ab98'
532 token = lexer.next_token
533 token.name.should == 'WS'
534 token.start.should == 12
535 token.stop.should == 14
536 token.text.should == " \n "
538 token = lexer.next_token
539 token.name.should == 'IDENTIFIER'
540 token.start.should == 15
541 token.stop.should == 20
542 token.text.should == 'A12sdf'
544 token = lexer.next_token
545 token.name.should == '<EOF>'
548 example "contains characters without a matching token" do
