1 2 from io import StringIO 3 import os 4 import unittest 5 import antlr3 6 7 8 class TestStringStream(unittest.TestCase): 9 """Test case for the StringStream class.""" 10 11 def testSize(self): 12 """StringStream.size()""" 13 14 stream = antlr3.StringStream('foo') 15 16 self.assertEqual(stream.size(), 3) 17 18 19 def testIndex(self): 20 """StringStream.index()""" 21 22 stream = antlr3.StringStream('foo') 23 24 self.assertEqual(stream.index(), 0) 25 26 27 def testConsume(self): 28 """StringStream.consume()""" 29 30 stream = antlr3.StringStream('foo\nbar') 31 32 stream.consume() # f 33 self.assertEqual(stream.index(), 1) 34 self.assertEqual(stream.charPositionInLine, 1) 35 self.assertEqual(stream.line, 1) 36 37 stream.consume() # o 38 self.assertEqual(stream.index(), 2) 39 self.assertEqual(stream.charPositionInLine, 2) 40 self.assertEqual(stream.line, 1) 41 42 stream.consume() # o 43 self.assertEqual(stream.index(), 3) 44 self.assertEqual(stream.charPositionInLine, 3) 45 self.assertEqual(stream.line, 1) 46 47 stream.consume() # \n 48 self.assertEqual(stream.index(), 4) 49 self.assertEqual(stream.charPositionInLine, 0) 50 self.assertEqual(stream.line, 2) 51 52 stream.consume() # b 53 self.assertEqual(stream.index(), 5) 54 self.assertEqual(stream.charPositionInLine, 1) 55 self.assertEqual(stream.line, 2) 56 57 stream.consume() # a 58 self.assertEqual(stream.index(), 6) 59 self.assertEqual(stream.charPositionInLine, 2) 60 self.assertEqual(stream.line, 2) 61 62 stream.consume() # r 63 self.assertEqual(stream.index(), 7) 64 self.assertEqual(stream.charPositionInLine, 3) 65 self.assertEqual(stream.line, 2) 66 67 stream.consume() # EOF 68 self.assertEqual(stream.index(), 7) 69 self.assertEqual(stream.charPositionInLine, 3) 70 self.assertEqual(stream.line, 2) 71 72 stream.consume() # EOF 73 self.assertEqual(stream.index(), 7) 74 self.assertEqual(stream.charPositionInLine, 3) 75 self.assertEqual(stream.line, 2) 76 77 78 def testReset(self): 79 """StringStream.reset()""" 80 81 stream = antlr3.StringStream('foo') 82 83 stream.consume() 84 stream.consume() 85 86 stream.reset() 87 self.assertEqual(stream.index(), 0) 88 self.assertEqual(stream.line, 1) 89 self.assertEqual(stream.charPositionInLine, 0) 90 self.assertEqual(stream.LT(1), 'f') 91 92 93 def testLA(self): 94 """StringStream.LA()""" 95 96 stream = antlr3.StringStream('foo') 97 98 self.assertEqual(stream.LT(1), 'f') 99 self.assertEqual(stream.LT(2), 'o') 100 self.assertEqual(stream.LT(3), 'o') 101 102 stream.consume() 103 stream.consume() 104 105 self.assertEqual(stream.LT(1), 'o') 106 self.assertEqual(stream.LT(2), antlr3.EOF) 107 self.assertEqual(stream.LT(3), antlr3.EOF) 108 109 110 def testSubstring(self): 111 """StringStream.substring()""" 112 113 stream = antlr3.StringStream('foobar') 114 115 self.assertEqual(stream.substring(0, 0), 'f') 116 self.assertEqual(stream.substring(0, 1), 'fo') 117 self.assertEqual(stream.substring(0, 5), 'foobar') 118 self.assertEqual(stream.substring(3, 5), 'bar') 119 120 121 def testSeekForward(self): 122 """StringStream.seek(): forward""" 123 124 stream = antlr3.StringStream('foo\nbar') 125 126 stream.seek(4) 127 128 self.assertEqual(stream.index(), 4) 129 self.assertEqual(stream.line, 2) 130 self.assertEqual(stream.charPositionInLine, 0) 131 self.assertEqual(stream.LT(1), 'b') 132 133 134 ## # not yet implemented 135 ## def testSeekBackward(self): 136 ## """StringStream.seek(): backward""" 137 138 ## stream = antlr3.StringStream('foo\nbar') 139 140 ## stream.seek(4) 141 ## stream.seek(1) 142 143 ## self.assertEqual(stream.index(), 1) 144 ## self.assertEqual(stream.line, 1) 145 ## self.assertEqual(stream.charPositionInLine, 1) 146 ## self.assertEqual(stream.LA(1), 'o') 147 148 149 def testMark(self): 150 """StringStream.mark()""" 151 152 stream = antlr3.StringStream('foo\nbar') 153 154 stream.seek(4) 155 156 marker = stream.mark() 157 self.assertEqual(marker, 1) 158 self.assertEqual(stream.markDepth, 1) 159 160 stream.consume() 161 marker = stream.mark() 162 self.assertEqual(marker, 2) 163 self.assertEqual(stream.markDepth, 2) 164 165 166 def testReleaseLast(self): 167 """StringStream.release(): last marker""" 168 169 stream = antlr3.StringStream('foo\nbar') 170 171 stream.seek(4) 172 marker1 = stream.mark() 173 174 stream.consume() 175 marker2 = stream.mark() 176 177 stream.release() 178 self.assertEqual(stream.markDepth, 1) 179 180 # release same marker again, nothing has changed 181 stream.release() 182 self.assertEqual(stream.markDepth, 1) 183 184 185 def testReleaseNested(self): 186 """StringStream.release(): nested""" 187 188 stream = antlr3.StringStream('foo\nbar') 189 190 stream.seek(4) 191 marker1 = stream.mark() 192 193 stream.consume() 194 marker2 = stream.mark() 195 196 stream.consume() 197 marker3 = stream.mark() 198 199 stream.release(marker2) 200 self.assertEqual(stream.markDepth, 1) 201 202 203 def testRewindLast(self): 204 """StringStream.rewind(): last marker""" 205 206 stream = antlr3.StringStream('foo\nbar') 207 208 stream.seek(4) 209 210 marker = stream.mark() 211 stream.consume() 212 stream.consume() 213 214 stream.rewind() 215 self.assertEqual(stream.markDepth, 0) 216 self.assertEqual(stream.index(), 4) 217 self.assertEqual(stream.line, 2) 218 self.assertEqual(stream.charPositionInLine, 0) 219 self.assertEqual(stream.LT(1), 'b') 220 221 222 def testRewindNested(self): 223 """StringStream.rewind(): nested""" 224 225 stream = antlr3.StringStream('foo\nbar') 226 227 stream.seek(4) 228 marker1 = stream.mark() 229 230 stream.consume() 231 marker2 = stream.mark() 232 233 stream.consume() 234 marker3 = stream.mark() 235 236 stream.rewind(marker2) 237 self.assertEqual(stream.markDepth, 1) 238 self.assertEqual(stream.index(), 5) 239 self.assertEqual(stream.line, 2) 240 self.assertEqual(stream.charPositionInLine, 1) 241 self.assertEqual(stream.LT(1), 'a') 242 243 244 class TestFileStream(unittest.TestCase): 245 """Test case for the FileStream class.""" 246 247 248 def testNoEncoding(self): 249 path = os.path.join(os.path.dirname(__file__), 'teststreams.input1') 250 251 stream = antlr3.FileStream(path) 252 253 stream.seek(4) 254 marker1 = stream.mark() 255 256 stream.consume() 257 marker2 = stream.mark() 258 259 stream.consume() 260 marker3 = stream.mark() 261 262 stream.rewind(marker2) 263 self.assertEqual(stream.markDepth, 1) 264 self.assertEqual(stream.index(), 5) 265 self.assertEqual(stream.line, 2) 266 self.assertEqual(stream.charPositionInLine, 1) 267 self.assertEqual(stream.LT(1), 'a') 268 self.assertEqual(stream.LA(1), ord('a')) 269 270 271 def testEncoded(self): 272 path = os.path.join(os.path.dirname(__file__), 'teststreams.input2') 273 274 stream = antlr3.FileStream(path) 275 276 stream.seek(4) 277 marker1 = stream.mark() 278 279 stream.consume() 280 marker2 = stream.mark() 281 282 stream.consume() 283 marker3 = stream.mark() 284 285 stream.rewind(marker2) 286 self.assertEqual(stream.markDepth, 1) 287 self.assertEqual(stream.index(), 5) 288 self.assertEqual(stream.line, 2) 289 self.assertEqual(stream.charPositionInLine, 1) 290 self.assertEqual(stream.LT(1), '') 291 self.assertEqual(stream.LA(1), ord('')) 292 293 294 295 class TestInputStream(unittest.TestCase): 296 """Test case for the InputStream class.""" 297 298 def testNoEncoding(self): 299 file = StringIO('foo\nbar') 300 301 stream = antlr3.InputStream(file) 302 303 stream.seek(4) 304 marker1 = stream.mark() 305 306 stream.consume() 307 marker2 = stream.mark() 308 309 stream.consume() 310 marker3 = stream.mark() 311 312 stream.rewind(marker2) 313 self.assertEqual(stream.markDepth, 1) 314 self.assertEqual(stream.index(), 5) 315 self.assertEqual(stream.line, 2) 316 self.assertEqual(stream.charPositionInLine, 1) 317 self.assertEqual(stream.LT(1), 'a') 318 self.assertEqual(stream.LA(1), ord('a')) 319 320 321 def testEncoded(self): 322 file = StringIO('foo\nbr') 323 324 stream = antlr3.InputStream(file) 325 326 stream.seek(4) 327 marker1 = stream.mark() 328 329 stream.consume() 330 marker2 = stream.mark() 331 332 stream.consume() 333 marker3 = stream.mark() 334 335 stream.rewind(marker2) 336 self.assertEqual(stream.markDepth, 1) 337 self.assertEqual(stream.index(), 5) 338 self.assertEqual(stream.line, 2) 339 self.assertEqual(stream.charPositionInLine, 1) 340 self.assertEqual(stream.LT(1), '') 341 self.assertEqual(stream.LA(1), ord('')) 342 343 344 class TestCommonTokenStream(unittest.TestCase): 345 """Test case for the StringStream class.""" 346 347 def setUp(self): 348 """Setup test fixure 349 350 The constructor of CommonTokenStream needs a token source. This 351 is a simple mock class providing just the nextToken() method. 352 353 """ 354 355 class MockSource(object): 356 def __init__(self): 357 self.tokens = [] 358 359 def makeEOFToken(self): 360 return antlr3.CommonToken(type=antlr3.EOF) 361 362 def nextToken(self): 363 if self.tokens: 364 return self.tokens.pop(0) 365 return None 366 367 self.source = MockSource() 368 369 370 def testInit(self): 371 """CommonTokenStream.__init__()""" 372 373 stream = antlr3.CommonTokenStream(self.source) 374 self.assertEqual(stream.index(), -1) 375 376 377 def testSetTokenSource(self): 378 """CommonTokenStream.setTokenSource()""" 379 380 stream = antlr3.CommonTokenStream(None) 381 stream.setTokenSource(self.source) 382 self.assertEqual(stream.index(), -1) 383 self.assertEqual(stream.channel, antlr3.DEFAULT_CHANNEL) 384 385 386 def testLTEmptySource(self): 387 """CommonTokenStream.LT(): EOF (empty source)""" 388 389 stream = antlr3.CommonTokenStream(self.source) 390 391 lt1 = stream.LT(1) 392 self.assertEqual(lt1.type, antlr3.EOF) 393 394 395 def testLT1(self): 396 """CommonTokenStream.LT(1)""" 397 398 self.source.tokens.append( 399 antlr3.CommonToken(type=12) 400 ) 401 402 stream = antlr3.CommonTokenStream(self.source) 403 404 lt1 = stream.LT(1) 405 self.assertEqual(lt1.type, 12) 406 407 408 def testLT1WithHidden(self): 409 """CommonTokenStream.LT(1): with hidden tokens""" 410 411 self.source.tokens.append( 412 antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL) 413 ) 414 415 self.source.tokens.append( 416 antlr3.CommonToken(type=13) 417 ) 418 419 stream = antlr3.CommonTokenStream(self.source) 420 421 lt1 = stream.LT(1) 422 self.assertEqual(lt1.type, 13) 423 424 425 def testLT2BeyondEnd(self): 426 """CommonTokenStream.LT(2): beyond end""" 427 428 self.source.tokens.append( 429 antlr3.CommonToken(type=12) 430 ) 431 432 self.source.tokens.append( 433 antlr3.CommonToken(type=13, channel=antlr3.HIDDEN_CHANNEL) 434 ) 435 436 stream = antlr3.CommonTokenStream(self.source) 437 438 lt1 = stream.LT(2) 439 self.assertEqual(lt1.type, antlr3.EOF) 440 441 442 # not yet implemented 443 def testLTNegative(self): 444 """CommonTokenStream.LT(-1): look back""" 445 446 self.source.tokens.append( 447 antlr3.CommonToken(type=12) 448 ) 449 450 self.source.tokens.append( 451 antlr3.CommonToken(type=13) 452 ) 453 454 stream = antlr3.CommonTokenStream(self.source) 455 stream.fillBuffer() 456 stream.consume() 457 458 lt1 = stream.LT(-1) 459 self.assertEqual(lt1.type, 12) 460 461 462 def testLB1(self): 463 """CommonTokenStream.LB(1)""" 464 465 self.source.tokens.append( 466 antlr3.CommonToken(type=12) 467 ) 468 469 self.source.tokens.append( 470 antlr3.CommonToken(type=13) 471 ) 472 473 stream = antlr3.CommonTokenStream(self.source) 474 stream.fillBuffer() 475 stream.consume() 476 477 self.assertEqual(stream.LB(1).type, 12) 478 479 480 def testLTZero(self): 481 """CommonTokenStream.LT(0)""" 482 483 self.source.tokens.append( 484 antlr3.CommonToken(type=12) 485 ) 486 487 self.source.tokens.append( 488 antlr3.CommonToken(type=13) 489 ) 490 491 stream = antlr3.CommonTokenStream(self.source) 492 493 lt1 = stream.LT(0) 494 self.assertIsNone(lt1) 495 496 497 def testLBBeyondBegin(self): 498 """CommonTokenStream.LB(-1): beyond begin""" 499 500 self.source.tokens.append( 501 antlr3.CommonToken(type=12) 502 ) 503 504 self.source.tokens.append( 505 antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL) 506 ) 507 508 self.source.tokens.append( 509 antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL) 510 ) 511 512 self.source.tokens.append( 513 antlr3.CommonToken(type=13) 514 ) 515 516 stream = antlr3.CommonTokenStream(self.source) 517 self.assertIsNone(stream.LB(1)) 518 519 stream.consume() 520 stream.consume() 521 self.assertIsNone(stream.LB(3)) 522 523 524 def testFillBuffer(self): 525 """CommonTokenStream.fillBuffer()""" 526 527 self.source.tokens.append( 528 antlr3.CommonToken(type=12) 529 ) 530 531 self.source.tokens.append( 532 antlr3.CommonToken(type=13) 533 ) 534 535 self.source.tokens.append( 536 antlr3.CommonToken(type=14) 537 ) 538 539 self.source.tokens.append( 540 antlr3.CommonToken(type=antlr3.EOF) 541 ) 542 543 stream = antlr3.CommonTokenStream(self.source) 544 stream.fillBuffer() 545 546 self.assertEqual(len(stream.tokens), 3) 547 self.assertEqual(stream.tokens[0].type, 12) 548 self.assertEqual(stream.tokens[1].type, 13) 549 self.assertEqual(stream.tokens[2].type, 14) 550 551 552 def testConsume(self): 553 """CommonTokenStream.consume()""" 554 555 self.source.tokens.append( 556 antlr3.CommonToken(type=12) 557 ) 558 559 self.source.tokens.append( 560 antlr3.CommonToken(type=13) 561 ) 562 563 self.source.tokens.append( 564 antlr3.CommonToken(type=antlr3.EOF) 565 ) 566 567 stream = antlr3.CommonTokenStream(self.source) 568 self.assertEqual(stream.LA(1), 12) 569 570 stream.consume() 571 self.assertEqual(stream.LA(1), 13) 572 573 stream.consume() 574 self.assertEqual(stream.LA(1), antlr3.EOF) 575 576 stream.consume() 577 self.assertEqual(stream.LA(1), antlr3.EOF) 578 579 580 def testSeek(self): 581 """CommonTokenStream.seek()""" 582 583 self.source.tokens.append( 584 antlr3.CommonToken(type=12) 585 ) 586 587 self.source.tokens.append( 588 antlr3.CommonToken(type=13) 589 ) 590 591 self.source.tokens.append( 592 antlr3.CommonToken(type=antlr3.EOF) 593 ) 594 595 stream = antlr3.CommonTokenStream(self.source) 596 self.assertEqual(stream.LA(1), 12) 597 598 stream.seek(2) 599 self.assertEqual(stream.LA(1), antlr3.EOF) 600 601 stream.seek(0) 602 self.assertEqual(stream.LA(1), 12) 603 604 605 def testMarkRewind(self): 606 """CommonTokenStream.mark()/rewind()""" 607 608 self.source.tokens.append( 609 antlr3.CommonToken(type=12) 610 ) 611 612 self.source.tokens.append( 613 antlr3.CommonToken(type=13) 614 ) 615 616 self.source.tokens.append( 617 antlr3.CommonToken(type=antlr3.EOF) 618 ) 619 620 stream = antlr3.CommonTokenStream(self.source) 621 stream.fillBuffer() 622 623 stream.consume() 624 marker = stream.mark() 625 626 stream.consume() 627 stream.rewind(marker) 628 629 self.assertEqual(stream.LA(1), 13) 630 631 632 def testToString(self): 633 """CommonTokenStream.toString()""" 634 635 self.source.tokens.append( 636 antlr3.CommonToken(type=12, text="foo") 637 ) 638 639 self.source.tokens.append( 640 antlr3.CommonToken(type=13, text="bar") 641 ) 642 643 self.source.tokens.append( 644 antlr3.CommonToken(type=14, text="gnurz") 645 ) 646 647 self.source.tokens.append( 648 antlr3.CommonToken(type=15, text="blarz") 649 ) 650 651 stream = antlr3.CommonTokenStream(self.source) 652 653 self.assertEqual(stream.toString(), "foobargnurzblarz") 654 self.assertEqual(stream.toString(1, 2), "bargnurz") 655 self.assertEqual(stream.toString(stream.tokens[1], stream.tokens[-2]), "bargnurz") 656 657 658 if __name__ == "__main__": 659 unittest.main(testRunner=unittest.TextTestRunner(verbosity=2)) 660
