Home | History | Annotate | Download | only in unittests 
      1 # -*- coding: utf-8 -*-
      2 
      3 import os
      4 import unittest
      5 from StringIO import StringIO
      6 import antlr3
      7 
      8 
      9 class TestStringStream(unittest.TestCase):
     10     """Test case for the StringStream class."""
     11 
     12     def testSize(self):
     13         """StringStream.size()"""
     14 
     15         stream = antlr3.StringStream('foo')
     16 
     17         self.failUnlessEqual(stream.size(), 3)
     18 
     19 
     20     def testIndex(self):
     21         """StringStream.index()"""
     22 
     23         stream = antlr3.StringStream('foo')
     24 
     25         self.failUnlessEqual(stream.index(), 0)
     26 
     27 
     28     def testConsume(self):
     29         """StringStream.consume()"""
     30 
     31         stream = antlr3.StringStream('foo\nbar')
     32 
     33         stream.consume() # f
     34         self.failUnlessEqual(stream.index(), 1)
     35         self.failUnlessEqual(stream.charPositionInLine, 1)
     36         self.failUnlessEqual(stream.line, 1)
     37 
     38         stream.consume() # o
     39         self.failUnlessEqual(stream.index(), 2)
     40         self.failUnlessEqual(stream.charPositionInLine, 2)
     41         self.failUnlessEqual(stream.line, 1)
     42 
     43         stream.consume() # o
     44         self.failUnlessEqual(stream.index(), 3)
     45         self.failUnlessEqual(stream.charPositionInLine, 3)
     46         self.failUnlessEqual(stream.line, 1)
     47 
     48         stream.consume() # \n
     49         self.failUnlessEqual(stream.index(), 4)
     50         self.failUnlessEqual(stream.charPositionInLine, 0)
     51         self.failUnlessEqual(stream.line, 2)
     52 
     53         stream.consume() # b
     54         self.failUnlessEqual(stream.index(), 5)
     55         self.failUnlessEqual(stream.charPositionInLine, 1)
     56         self.failUnlessEqual(stream.line, 2)
     57 
     58         stream.consume() # a
     59         self.failUnlessEqual(stream.index(), 6)
     60         self.failUnlessEqual(stream.charPositionInLine, 2)
     61         self.failUnlessEqual(stream.line, 2)
     62 
     63         stream.consume() # r
     64         self.failUnlessEqual(stream.index(), 7)
     65         self.failUnlessEqual(stream.charPositionInLine, 3)
     66         self.failUnlessEqual(stream.line, 2)
     67 
     68         stream.consume() # EOF
     69         self.failUnlessEqual(stream.index(), 7)
     70         self.failUnlessEqual(stream.charPositionInLine, 3)
     71         self.failUnlessEqual(stream.line, 2)
     72 
     73         stream.consume() # EOF
     74         self.failUnlessEqual(stream.index(), 7)
     75         self.failUnlessEqual(stream.charPositionInLine, 3)
     76         self.failUnlessEqual(stream.line, 2)
     77 
     78 
     79     def testReset(self):
     80         """StringStream.reset()"""
     81 
     82         stream = antlr3.StringStream('foo')
     83 
     84         stream.consume()
     85         stream.consume()
     86 
     87         stream.reset()
     88         self.failUnlessEqual(stream.index(), 0)
     89         self.failUnlessEqual(stream.line, 1)
     90         self.failUnlessEqual(stream.charPositionInLine, 0)
     91         self.failUnlessEqual(stream.LT(1), 'f')
     92 
     93 
     94     def testLA(self):
     95         """StringStream.LA()"""
     96 
     97         stream = antlr3.StringStream('foo')
     98 
     99         self.failUnlessEqual(stream.LT(1), 'f')
    100         self.failUnlessEqual(stream.LT(2), 'o')
    101         self.failUnlessEqual(stream.LT(3), 'o')
    102 
    103         stream.consume()
    104         stream.consume()
    105 
    106         self.failUnlessEqual(stream.LT(1), 'o')
    107         self.failUnlessEqual(stream.LT(2), antlr3.EOF)
    108         self.failUnlessEqual(stream.LT(3), antlr3.EOF)
    109 
    110 
    111     def testSubstring(self):
    112         """StringStream.substring()"""
    113 
    114         stream = antlr3.StringStream('foobar')
    115 
    116         self.failUnlessEqual(stream.substring(0, 0), 'f')
    117         self.failUnlessEqual(stream.substring(0, 1), 'fo')
    118         self.failUnlessEqual(stream.substring(0, 5), 'foobar')
    119         self.failUnlessEqual(stream.substring(3, 5), 'bar')
    120 
    121 
    122     def testSeekForward(self):
    123         """StringStream.seek(): forward"""
    124 
    125         stream = antlr3.StringStream('foo\nbar')
    126 
    127         stream.seek(4)
    128 
    129         self.failUnlessEqual(stream.index(), 4)
    130         self.failUnlessEqual(stream.line, 2)
    131         self.failUnlessEqual(stream.charPositionInLine, 0)
    132         self.failUnlessEqual(stream.LT(1), 'b')
    133 
    134 
    135 ##     # not yet implemented
    136 ##     def testSeekBackward(self):
    137 ##         """StringStream.seek(): backward"""
    138 
    139 ##         stream = antlr3.StringStream('foo\nbar')
    140 
    141 ##         stream.seek(4)
    142 ##         stream.seek(1)
    143 
    144 ##         self.failUnlessEqual(stream.index(), 1)
    145 ##         self.failUnlessEqual(stream.line, 1)
    146 ##         self.failUnlessEqual(stream.charPositionInLine, 1)
    147 ##         self.failUnlessEqual(stream.LA(1), 'o')
    148 
    149 
    150     def testMark(self):
    151         """StringStream.mark()"""
    152 
    153         stream = antlr3.StringStream('foo\nbar')
    154 
    155         stream.seek(4)
    156 
    157         marker = stream.mark()
    158         self.failUnlessEqual(marker, 1)
    159         self.failUnlessEqual(stream.markDepth, 1)
    160 
    161         stream.consume()
    162         marker = stream.mark()
    163         self.failUnlessEqual(marker, 2)
    164         self.failUnlessEqual(stream.markDepth, 2)
    165 
    166 
    167     def testReleaseLast(self):
    168         """StringStream.release(): last marker"""
    169 
    170         stream = antlr3.StringStream('foo\nbar')
    171 
    172         stream.seek(4)
    173         marker1 = stream.mark()
    174 
    175         stream.consume()
    176         marker2 = stream.mark()
    177 
    178         stream.release()
    179         self.failUnlessEqual(stream.markDepth, 1)
    180 
    181         # release same marker again, nothing has changed
    182         stream.release()
    183         self.failUnlessEqual(stream.markDepth, 1)
    184 
    185 
    186     def testReleaseNested(self):
    187         """StringStream.release(): nested"""
    188 
    189         stream = antlr3.StringStream('foo\nbar')
    190 
    191         stream.seek(4)
    192         marker1 = stream.mark()
    193 
    194         stream.consume()
    195         marker2 = stream.mark()
    196 
    197         stream.consume()
    198         marker3 = stream.mark()
    199 
    200         stream.release(marker2)
    201         self.failUnlessEqual(stream.markDepth, 1)
    202 
    203 
    204     def testRewindLast(self):
    205         """StringStream.rewind(): last marker"""
    206 
    207         stream = antlr3.StringStream('foo\nbar')
    208 
    209         stream.seek(4)
    210 
    211         marker = stream.mark()
    212         stream.consume()
    213         stream.consume()
    214 
    215         stream.rewind()
    216         self.failUnlessEqual(stream.markDepth, 0)
    217         self.failUnlessEqual(stream.index(), 4)
    218         self.failUnlessEqual(stream.line, 2)
    219         self.failUnlessEqual(stream.charPositionInLine, 0)
    220         self.failUnlessEqual(stream.LT(1), 'b')
    221 
    222 
    223     def testRewindNested(self):
    224         """StringStream.rewind(): nested"""
    225 
    226         stream = antlr3.StringStream('foo\nbar')
    227 
    228         stream.seek(4)
    229         marker1 = stream.mark()
    230 
    231         stream.consume()
    232         marker2 = stream.mark()
    233 
    234         stream.consume()
    235         marker3 = stream.mark()
    236 
    237         stream.rewind(marker2)
    238         self.failUnlessEqual(stream.markDepth, 1)
    239         self.failUnlessEqual(stream.index(), 5)
    240         self.failUnlessEqual(stream.line, 2)
    241         self.failUnlessEqual(stream.charPositionInLine, 1)
    242         self.failUnlessEqual(stream.LT(1), 'a')
    243 
    244 
    245 class TestFileStream(unittest.TestCase):
    246     """Test case for the FileStream class."""
    247 
    248 
    249     def testNoEncoding(self):
    250         path = os.path.join(os.path.dirname(__file__), 'teststreams.input1')
    251 
    252         stream = antlr3.FileStream(path)
    253 
    254         stream.seek(4)
    255         marker1 = stream.mark()
    256 
    257         stream.consume()
    258         marker2 = stream.mark()
    259 
    260         stream.consume()
    261         marker3 = stream.mark()
    262 
    263         stream.rewind(marker2)
    264         self.failUnlessEqual(stream.markDepth, 1)
    265         self.failUnlessEqual(stream.index(), 5)
    266         self.failUnlessEqual(stream.line, 2)
    267         self.failUnlessEqual(stream.charPositionInLine, 1)
    268         self.failUnlessEqual(stream.LT(1), 'a')
    269         self.failUnlessEqual(stream.LA(1), ord('a'))
    270 
    271 
    272     def testEncoded(self):
    273         path = os.path.join(os.path.dirname(__file__), 'teststreams.input2')
    274 
    275         stream = antlr3.FileStream(path, 'utf-8')
    276 
    277         stream.seek(4)
    278         marker1 = stream.mark()
    279 
    280         stream.consume()
    281         marker2 = stream.mark()
    282 
    283         stream.consume()
    284         marker3 = stream.mark()
    285 
    286         stream.rewind(marker2)
    287         self.failUnlessEqual(stream.markDepth, 1)
    288         self.failUnlessEqual(stream.index(), 5)
    289         self.failUnlessEqual(stream.line, 2)
    290         self.failUnlessEqual(stream.charPositionInLine, 1)
    291         self.failUnlessEqual(stream.LT(1), u'')
    292         self.failUnlessEqual(stream.LA(1), ord(u''))
    293 
    294 
    295 
    296 class TestInputStream(unittest.TestCase):
    297     """Test case for the InputStream class."""
    298 
    299     def testNoEncoding(self):
    300         file = StringIO('foo\nbar')
    301 
    302         stream = antlr3.InputStream(file)
    303 
    304         stream.seek(4)
    305         marker1 = stream.mark()
    306 
    307         stream.consume()
    308         marker2 = stream.mark()
    309 
    310         stream.consume()
    311         marker3 = stream.mark()
    312 
    313         stream.rewind(marker2)
    314         self.failUnlessEqual(stream.markDepth, 1)
    315         self.failUnlessEqual(stream.index(), 5)
    316         self.failUnlessEqual(stream.line, 2)
    317         self.failUnlessEqual(stream.charPositionInLine, 1)
    318         self.failUnlessEqual(stream.LT(1), 'a')
    319         self.failUnlessEqual(stream.LA(1), ord('a'))
    320 
    321 
    322     def testEncoded(self):
    323         file = StringIO(u'foo\nbr'.encode('utf-8'))
    324 
    325         stream = antlr3.InputStream(file, 'utf-8')
    326 
    327         stream.seek(4)
    328         marker1 = stream.mark()
    329 
    330         stream.consume()
    331         marker2 = stream.mark()
    332 
    333         stream.consume()
    334         marker3 = stream.mark()
    335 
    336         stream.rewind(marker2)
    337         self.failUnlessEqual(stream.markDepth, 1)
    338         self.failUnlessEqual(stream.index(), 5)
    339         self.failUnlessEqual(stream.line, 2)
    340         self.failUnlessEqual(stream.charPositionInLine, 1)
    341         self.failUnlessEqual(stream.LT(1), u'')
    342         self.failUnlessEqual(stream.LA(1), ord(u''))
    343 
    344 
    345 class TestCommonTokenStream(unittest.TestCase):
    346     """Test case for the StringStream class."""
    347 
    348     def setUp(self):
    349         """Setup test fixure
    350 
    351         The constructor of CommonTokenStream needs a token source. This
    352         is a simple mock class providing just the nextToken() method.
    353 
    354         """
    355 
    356         class MockSource(object):
    357             def __init__(self):
    358                 self.tokens = []
    359 
    360             def makeEOFToken(self):
    361                 return antlr3.CommonToken(type=antlr3.EOF)
    362 
    363             def nextToken(self):
    364                 try:
    365                     return self.tokens.pop(0)
    366                 except IndexError:
    367                     return None
    368 
    369         self.source = MockSource()
    370 
    371 
    372     def testInit(self):
    373         """CommonTokenStream.__init__()"""
    374 
    375         stream = antlr3.CommonTokenStream(self.source)
    376         self.failUnlessEqual(stream.index(), -1)
    377 
    378 
    379     def testSetTokenSource(self):
    380         """CommonTokenStream.setTokenSource()"""
    381 
    382         stream = antlr3.CommonTokenStream(None)
    383         stream.setTokenSource(self.source)
    384         self.failUnlessEqual(stream.index(), -1)
    385         self.failUnlessEqual(stream.channel, antlr3.DEFAULT_CHANNEL)
    386 
    387 
    388     def testLTEmptySource(self):
    389         """CommonTokenStream.LT(): EOF (empty source)"""
    390 
    391         stream = antlr3.CommonTokenStream(self.source)
    392 
    393         lt1 = stream.LT(1)
    394         self.failUnlessEqual(lt1.type, antlr3.EOF)
    395 
    396 
    397     def testLT1(self):
    398         """CommonTokenStream.LT(1)"""
    399 
    400         self.source.tokens.append(
    401             antlr3.CommonToken(type=12)
    402             )
    403 
    404         stream = antlr3.CommonTokenStream(self.source)
    405 
    406         lt1 = stream.LT(1)
    407         self.failUnlessEqual(lt1.type, 12)
    408 
    409 
    410     def testLT1WithHidden(self):
    411         """CommonTokenStream.LT(1): with hidden tokens"""
    412 
    413         self.source.tokens.append(
    414             antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL)
    415             )
    416 
    417         self.source.tokens.append(
    418             antlr3.CommonToken(type=13)
    419             )
    420 
    421         stream = antlr3.CommonTokenStream(self.source)
    422 
    423         lt1 = stream.LT(1)
    424         self.failUnlessEqual(lt1.type, 13)
    425 
    426 
    427     def testLT2BeyondEnd(self):
    428         """CommonTokenStream.LT(2): beyond end"""
    429 
    430         self.source.tokens.append(
    431             antlr3.CommonToken(type=12)
    432             )
    433 
    434         self.source.tokens.append(
    435             antlr3.CommonToken(type=13, channel=antlr3.HIDDEN_CHANNEL)
    436             )
    437 
    438         stream = antlr3.CommonTokenStream(self.source)
    439 
    440         lt1 = stream.LT(2)
    441         self.failUnlessEqual(lt1.type, antlr3.EOF)
    442 
    443 
    444     # not yet implemented
    445     def testLTNegative(self):
    446         """CommonTokenStream.LT(-1): look back"""
    447 
    448         self.source.tokens.append(
    449             antlr3.CommonToken(type=12)
    450             )
    451 
    452         self.source.tokens.append(
    453             antlr3.CommonToken(type=13)
    454             )
    455 
    456         stream = antlr3.CommonTokenStream(self.source)
    457         stream.fillBuffer()
    458         stream.consume()
    459 
    460         lt1 = stream.LT(-1)
    461         self.failUnlessEqual(lt1.type, 12)
    462 
    463 
    464     def testLB1(self):
    465         """CommonTokenStream.LB(1)"""
    466 
    467         self.source.tokens.append(
    468             antlr3.CommonToken(type=12)
    469             )
    470 
    471         self.source.tokens.append(
    472             antlr3.CommonToken(type=13)
    473             )
    474 
    475         stream = antlr3.CommonTokenStream(self.source)
    476         stream.fillBuffer()
    477         stream.consume()
    478 
    479         self.failUnlessEqual(stream.LB(1).type, 12)
    480 
    481 
    482     def testLTZero(self):
    483         """CommonTokenStream.LT(0)"""
    484 
    485         self.source.tokens.append(
    486             antlr3.CommonToken(type=12)
    487             )
    488 
    489         self.source.tokens.append(
    490             antlr3.CommonToken(type=13)
    491             )
    492 
    493         stream = antlr3.CommonTokenStream(self.source)
    494 
    495         lt1 = stream.LT(0)
    496         self.failUnless(lt1 is None)
    497 
    498 
    499     def testLBBeyondBegin(self):
    500         """CommonTokenStream.LB(-1): beyond begin"""
    501 
    502         self.source.tokens.append(
    503             antlr3.CommonToken(type=12)
    504             )
    505 
    506         self.source.tokens.append(
    507             antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL)
    508             )
    509 
    510         self.source.tokens.append(
    511             antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL)
    512             )
    513 
    514         self.source.tokens.append(
    515             antlr3.CommonToken(type=13)
    516             )
    517 
    518         stream = antlr3.CommonTokenStream(self.source)
    519         self.failUnless(stream.LB(1) is None)
    520 
    521         stream.consume()
    522         stream.consume()
    523         self.failUnless(stream.LB(3) is None)
    524 
    525 
    526     def testFillBuffer(self):
    527         """CommonTokenStream.fillBuffer()"""
    528 
    529         self.source.tokens.append(
    530             antlr3.CommonToken(type=12)
    531             )
    532 
    533         self.source.tokens.append(
    534             antlr3.CommonToken(type=13)
    535             )
    536 
    537         self.source.tokens.append(
    538             antlr3.CommonToken(type=14)
    539             )
    540 
    541         self.source.tokens.append(
    542             antlr3.CommonToken(type=antlr3.EOF)
    543             )
    544 
    545         stream = antlr3.CommonTokenStream(self.source)
    546         stream.fillBuffer()
    547 
    548         self.failUnlessEqual(len(stream.tokens), 3)
    549         self.failUnlessEqual(stream.tokens[0].type, 12)
    550         self.failUnlessEqual(stream.tokens[1].type, 13)
    551         self.failUnlessEqual(stream.tokens[2].type, 14)
    552 
    553 
    554     def testConsume(self):
    555         """CommonTokenStream.consume()"""
    556 
    557         self.source.tokens.append(
    558             antlr3.CommonToken(type=12)
    559             )
    560 
    561         self.source.tokens.append(
    562             antlr3.CommonToken(type=13)
    563             )
    564 
    565         self.source.tokens.append(
    566             antlr3.CommonToken(type=antlr3.EOF)
    567             )
    568 
    569         stream = antlr3.CommonTokenStream(self.source)
    570         self.failUnlessEqual(stream.LA(1), 12)
    571 
    572         stream.consume()
    573         self.failUnlessEqual(stream.LA(1), 13)
    574 
    575         stream.consume()
    576         self.failUnlessEqual(stream.LA(1), antlr3.EOF)
    577 
    578         stream.consume()
    579         self.failUnlessEqual(stream.LA(1), antlr3.EOF)
    580 
    581 
    582     def testSeek(self):
    583         """CommonTokenStream.seek()"""
    584 
    585         self.source.tokens.append(
    586             antlr3.CommonToken(type=12)
    587             )
    588 
    589         self.source.tokens.append(
    590             antlr3.CommonToken(type=13)
    591             )
    592 
    593         self.source.tokens.append(
    594             antlr3.CommonToken(type=antlr3.EOF)
    595             )
    596 
    597         stream = antlr3.CommonTokenStream(self.source)
    598         self.failUnlessEqual(stream.LA(1), 12)
    599 
    600         stream.seek(2)
    601         self.failUnlessEqual(stream.LA(1), antlr3.EOF)
    602 
    603         stream.seek(0)
    604         self.failUnlessEqual(stream.LA(1), 12)
    605 
    606 
    607     def testMarkRewind(self):
    608         """CommonTokenStream.mark()/rewind()"""
    609 
    610         self.source.tokens.append(
    611             antlr3.CommonToken(type=12)
    612             )
    613 
    614         self.source.tokens.append(
    615             antlr3.CommonToken(type=13)
    616             )
    617 
    618         self.source.tokens.append(
    619             antlr3.CommonToken(type=antlr3.EOF)
    620             )
    621 
    622         stream = antlr3.CommonTokenStream(self.source)
    623         stream.fillBuffer()
    624 
    625         stream.consume()
    626         marker = stream.mark()
    627 
    628         stream.consume()
    629         stream.rewind(marker)
    630 
    631         self.failUnlessEqual(stream.LA(1), 13)
    632 
    633 
    634     def testToString(self):
    635         """CommonTokenStream.toString()"""
    636 
    637         self.source.tokens.append(
    638             antlr3.CommonToken(type=12, text="foo")
    639             )
    640 
    641         self.source.tokens.append(
    642             antlr3.CommonToken(type=13, text="bar")
    643             )
    644 
    645         self.source.tokens.append(
    646             antlr3.CommonToken(type=14, text="gnurz")
    647             )
    648 
    649         self.source.tokens.append(
    650             antlr3.CommonToken(type=15, text="blarz")
    651             )
    652 
    653         stream = antlr3.CommonTokenStream(self.source)
    654 
    655         assert stream.toString() == "foobargnurzblarz"
    656         assert stream.toString(1, 2) == "bargnurz"
    657         assert stream.toString(stream.tokens[1], stream.tokens[-2]) == "bargnurz"
    658 
    659 
    660 if __name__ == "__main__":
    661     unittest.main(testRunner=unittest.TextTestRunner(verbosity=2))
    662