Home | History | Annotate | Download | only in tests 
      1 import unittest
      2 import textwrap
      3 import antlr3
      4 import antlr3.tree
      5 import testbase
      6 
      7 class T(testbase.ANTLRTest):
      8     def walkerClass(self, base):
      9         class TWalker(base):
     10             def __init__(self, *args, **kwargs):
     11                 base.__init__(self, *args, **kwargs)
     12 
     13                 self._output = ""
     14 
     15 
     16             def capture(self, t):
     17                 self._output += t
     18 
     19 
     20             def traceIn(self, ruleName, ruleIndex):
     21                 self.traces.append('>'+ruleName)
     22 
     23 
     24             def traceOut(self, ruleName, ruleIndex):
     25                 self.traces.append('<'+ruleName)
     26 
     27 
     28             def recover(self, input, re):
     29                 # no error recovery yet, just crash!
     30                 raise
     31             
     32         return TWalker
     33     
     34 
     35     def execTreeParser(self, grammar, grammarEntry, treeGrammar, treeEntry, input):
     36         lexerCls, parserCls = self.compileInlineGrammar(grammar)
     37         walkerCls = self.compileInlineGrammar(treeGrammar)
     38 
     39         cStream = antlr3.StringStream(input)
     40         lexer = lexerCls(cStream)
     41         tStream = antlr3.CommonTokenStream(lexer)
     42         parser = parserCls(tStream)
     43         r = getattr(parser, grammarEntry)()
     44         nodes = antlr3.tree.CommonTreeNodeStream(r.tree)
     45         nodes.setTokenStream(tStream)
     46         walker = walkerCls(nodes)
     47         getattr(walker, treeEntry)()
     48 
     49         return walker._output
     50     
     51 
     52     def testFlatList(self):
     53         grammar = textwrap.dedent(
     54         r'''grammar T;
     55         options {
     56             language=Python;
     57             output=AST;
     58         }
     59         a : ID INT;
     60         ID : 'a'..'z'+ ;
     61         INT : '0'..'9'+;
     62         WS : (' '|'\n') {$channel=HIDDEN;} ;
     63         ''')
     64         
     65         treeGrammar = textwrap.dedent(
     66         r'''tree grammar TP;
     67         options {
     68             language=Python;
     69             ASTLabelType=CommonTree;
     70         }
     71         a : ID INT
     72             {self.capture("\%s, \%s" \% ($ID, $INT))}
     73           ;
     74         ''')
     75 
     76         found = self.execTreeParser(
     77             grammar, 'a',
     78             treeGrammar, 'a',
     79             "abc 34"
     80             )
     81 
     82         self.failUnlessEqual("abc, 34", found)
     83         
     84 
     85 
     86     def testSimpleTree(self):
     87         grammar = textwrap.dedent(
     88             r'''grammar T;
     89             options {
     90                 language=Python;
     91                 output=AST;
     92             }
     93             a : ID INT -> ^(ID INT);
     94             ID : 'a'..'z'+ ;
     95             INT : '0'..'9'+;
     96             WS : (' '|'\\n') {$channel=HIDDEN;} ;
     97             ''')
     98 
     99         treeGrammar = textwrap.dedent(
    100             r'''tree grammar TP;
    101             options {
    102                 language=Python;
    103                 ASTLabelType=CommonTree;
    104             }
    105             a : ^(ID INT)
    106                 {self.capture(str($ID)+", "+str($INT))}
    107               ;
    108             ''')
    109 
    110         found = self.execTreeParser(
    111             grammar, 'a',
    112             treeGrammar, 'a',
    113             "abc 34"
    114             )
    115             
    116         self.failUnlessEqual("abc, 34", found)
    117 
    118 
    119     def testFlatVsTreeDecision(self):
    120         grammar = textwrap.dedent(
    121             r'''grammar T;
    122             options {
    123                 language=Python;
    124                 output=AST;
    125             }
    126             a : b c ;
    127             b : ID INT -> ^(ID INT);
    128             c : ID INT;
    129             ID : 'a'..'z'+ ;
    130             INT : '0'..'9'+;
    131             WS : (' '|'\\n') {$channel=HIDDEN;} ;
    132             ''')
    133         
    134         treeGrammar = textwrap.dedent(
    135             r'''tree grammar TP;
    136             options {
    137                 language=Python;
    138                 ASTLabelType=CommonTree;
    139             }
    140             a : b b ;
    141             b : ID INT    {self.capture(str($ID)+" "+str($INT)+'\n')}
    142               | ^(ID INT) {self.capture("^("+str($ID)+" "+str($INT)+')');}
    143               ;
    144             ''')
    145         
    146         found = self.execTreeParser(
    147             grammar, 'a',
    148             treeGrammar, 'a',
    149             "a 1 b 2"
    150             )
    151         self.failUnlessEqual("^(a 1)b 2\n", found)
    152 
    153 
    154     def testFlatVsTreeDecision2(self):
    155         grammar = textwrap.dedent(
    156             r"""grammar T;
    157             options {
    158                 language=Python;
    159                 output=AST;
    160             }
    161             a : b c ;
    162             b : ID INT+ -> ^(ID INT+);
    163             c : ID INT+;
    164             ID : 'a'..'z'+ ;
    165             INT : '0'..'9'+;
    166             WS : (' '|'\n') {$channel=HIDDEN;} ;
    167             """)
    168 
    169         treeGrammar = textwrap.dedent(
    170             r'''tree grammar TP;
    171             options {
    172                 language=Python;
    173                 ASTLabelType=CommonTree;
    174             }
    175             a : b b ;
    176             b : ID INT+    {self.capture(str($ID)+" "+str($INT)+"\n")}
    177               | ^(x=ID (y=INT)+) {self.capture("^("+str($x)+' '+str($y)+')')}
    178               ;
    179             ''')
    180 
    181         found = self.execTreeParser(
    182             grammar, 'a',
    183             treeGrammar, 'a',
    184             "a 1 2 3 b 4 5"
    185             )
    186         self.failUnlessEqual("^(a 3)b 5\n", found)
    187 
    188 
    189     def testCyclicDFALookahead(self):
    190         grammar = textwrap.dedent(
    191             r'''grammar T;
    192             options {
    193                 language=Python;
    194                 output=AST;
    195             }
    196             a : ID INT+ PERIOD;
    197             ID : 'a'..'z'+ ;
    198             INT : '0'..'9'+;
    199             SEMI : ';' ;
    200             PERIOD : '.' ;
    201             WS : (' '|'\n') {$channel=HIDDEN;} ;
    202             ''')
    203 
    204         treeGrammar = textwrap.dedent(
    205             r'''tree grammar TP;
    206             options {
    207                 language=Python;
    208                 ASTLabelType=CommonTree;
    209             }
    210             a : ID INT+ PERIOD {self.capture("alt 1")}
    211               | ID INT+ SEMI   {self.capture("alt 2")}
    212               ;
    213             ''')
    214 
    215         found = self.execTreeParser(
    216             grammar, 'a',
    217             treeGrammar, 'a',
    218             "a 1 2 3."
    219             )
    220         self.failUnlessEqual("alt 1", found)
    221 
    222 
    223 ##     def testTemplateOutput(self):
    224 ## 		String grammar =
    225 ## 			"grammar T;\n" +
    226 ## 			"options {output=AST;}\n" +
    227 ## 			"a : ID INT;\n" +
    228 ## 			"ID : 'a'..'z'+ ;\n" +
    229 ## 			"INT : '0'..'9'+;\n" +
    230 ## 			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
    231 
    232 ## 		String treeGrammar =
    233 ## 			"tree grammar TP;\n" +
    234 ## 			"options {output=template; ASTLabelType=CommonTree;}\n" +
    235 ## 			"s : a {System.out.println($a.st);};\n" +
    236 ## 			"a : ID INT -> {new StringTemplate($INT.text)}\n" +
    237 ## 			"  ;\n";
    238 
    239 ## 		String found = execTreeParser("T.g", grammar, "TParser", "TP.g",
    240 ## 				    treeGrammar, "TP", "TLexer", "a", "s", "abc 34");
    241 ## 		assertEquals("34\n", found);
    242 ## 	}
    243 
    244 
    245     def testNullableChildList(self):
    246         grammar = textwrap.dedent(
    247             r'''grammar T;
    248             options {
    249                 language=Python;
    250                 output=AST;
    251             }
    252             a : ID INT? -> ^(ID INT?);
    253             ID : 'a'..'z'+ ;
    254             INT : '0'..'9'+;
    255             WS : (' '|'\\n') {$channel=HIDDEN;} ;
    256             ''')
    257         
    258         treeGrammar = textwrap.dedent(
    259             r'''tree grammar TP;
    260             options {
    261                 language=Python;
    262                 ASTLabelType=CommonTree;
    263             }
    264             a : ^(ID INT?)
    265                 {self.capture(str($ID))}
    266               ;
    267             ''')
    268 
    269         found = self.execTreeParser(
    270             grammar, 'a',
    271             treeGrammar, 'a',
    272             "abc"
    273             )
    274         self.failUnlessEqual("abc", found)
    275 
    276 
    277     def testNullableChildList2(self):
    278         grammar = textwrap.dedent(
    279             r'''grammar T;
    280             options {
    281                 language=Python;
    282                 output=AST;
    283             }
    284             a : ID INT? SEMI -> ^(ID INT?) SEMI ;
    285             ID : 'a'..'z'+ ;
    286             INT : '0'..'9'+;
    287             SEMI : ';' ;
    288             WS : (' '|'\n') {$channel=HIDDEN;} ;
    289             ''')
    290 
    291         treeGrammar = textwrap.dedent(
    292             r'''tree grammar TP;
    293             options {
    294                 language=Python;
    295                 ASTLabelType=CommonTree;
    296             }
    297             a : ^(ID INT?) SEMI
    298                 {self.capture(str($ID))}
    299               ;
    300             ''')
    301         
    302         found = self.execTreeParser(
    303             grammar, 'a',
    304             treeGrammar, 'a',
    305             "abc;"
    306             )
    307         self.failUnlessEqual("abc", found)
    308 
    309 
    310     def testNullableChildList3(self):
    311         grammar = textwrap.dedent(
    312             r'''grammar T;
    313             options {
    314                 language=Python;
    315                 output=AST;
    316             }
    317             a : x=ID INT? (y=ID)? SEMI -> ^($x INT? $y?) SEMI ;
    318             ID : 'a'..'z'+ ;
    319             INT : '0'..'9'+;
    320             SEMI : ';' ;
    321             WS : (' '|'\\n') {$channel=HIDDEN;} ;
    322             ''')
    323 
    324         treeGrammar = textwrap.dedent(
    325             r'''tree grammar TP;
    326             options {
    327                 language=Python;
    328                 ASTLabelType=CommonTree;
    329             }
    330             a : ^(ID INT? b) SEMI
    331                 {self.capture(str($ID)+", "+str($b.text))}
    332               ;
    333             b : ID? ;
    334             ''')
    335         
    336         found = self.execTreeParser(
    337             grammar, 'a',
    338             treeGrammar, 'a',
    339             "abc def;"
    340             )
    341         self.failUnlessEqual("abc, def", found)
    342 
    343 
    344     def testActionsAfterRoot(self):
    345         grammar = textwrap.dedent(
    346             r'''grammar T;
    347             options {
    348                 language=Python;
    349                 output=AST;
    350             }
    351             a : x=ID INT? SEMI -> ^($x INT?) ;
    352             ID : 'a'..'z'+ ;
    353             INT : '0'..'9'+;
    354             SEMI : ';' ;
    355             WS : (' '|'\n') {$channel=HIDDEN;} ;
    356             ''')
    357 
    358         treeGrammar = textwrap.dedent(
    359             r'''tree grammar TP;
    360             options {
    361                 language=Python;
    362                 ASTLabelType=CommonTree;
    363             }
    364             a @init {x=0} : ^(ID {x=1} {x=2} INT?)
    365                 {self.capture(str($ID)+", "+str(x))}
    366               ;
    367             ''')
    368 
    369         found = self.execTreeParser(
    370             grammar, 'a',
    371             treeGrammar, 'a',
    372             "abc;"
    373             )
    374         self.failUnless("abc, 2\n", found)
    375 
    376 
    377     def testWildcardLookahead(self):
    378         grammar = textwrap.dedent(
    379             r'''
    380             grammar T;
    381             options {language=Python; output=AST;}
    382             a : ID '+'^ INT;
    383             ID : 'a'..'z'+ ;
    384             INT : '0'..'9'+;
    385             SEMI : ';' ;
    386             PERIOD : '.' ;
    387             WS : (' '|'\n') {$channel=HIDDEN;} ;
    388             ''')
    389 
    390         treeGrammar = textwrap.dedent(
    391             r'''
    392             tree grammar TP; 
    393             options {language=Python; tokenVocab=T; ASTLabelType=CommonTree;}
    394             a : ^('+' . INT) { self.capture("alt 1") }
    395               ;
    396             ''')
    397 
    398         found = self.execTreeParser(
    399             grammar, 'a',
    400             treeGrammar, 'a',
    401             "a + 2")
    402         self.assertEquals("alt 1", found)
    403 
    404 
    405     def testWildcardLookahead2(self):
    406         grammar = textwrap.dedent(
    407             r'''
    408             grammar T;
    409             options {language=Python; output=AST;}
    410             a : ID '+'^ INT;
    411             ID : 'a'..'z'+ ;
    412             INT : '0'..'9'+;
    413             SEMI : ';' ;
    414             PERIOD : '.' ;
    415             WS : (' '|'\n') {$channel=HIDDEN;} ;
    416             ''')
    417 
    418         treeGrammar = textwrap.dedent(
    419             r'''
    420             tree grammar TP;
    421             options {language=Python; tokenVocab=T; ASTLabelType=CommonTree;}
    422             a : ^('+' . INT) { self.capture("alt 1") }
    423               | ^('+' . .)   { self.capture("alt 2") }
    424               ;
    425             ''')
    426 
    427         # AMBIG upon '+' DOWN INT UP etc.. but so what.
    428 
    429         found = self.execTreeParser(
    430             grammar, 'a',
    431             treeGrammar, 'a',
    432             "a + 2")
    433         self.assertEquals("alt 1", found)
    434 
    435 
    436     def testWildcardLookahead3(self):
    437         grammar = textwrap.dedent(
    438             r'''
    439             grammar T;
    440             options {language=Python; output=AST;}
    441             a : ID '+'^ INT;
    442             ID : 'a'..'z'+ ;
    443             INT : '0'..'9'+;
    444             SEMI : ';' ;
    445             PERIOD : '.' ;
    446             WS : (' '|'\n') {$channel=HIDDEN;} ;
    447             ''')
    448 
    449         treeGrammar = textwrap.dedent(
    450             r'''
    451             tree grammar TP;
    452             options {language=Python; tokenVocab=T; ASTLabelType=CommonTree;}
    453             a : ^('+' ID INT) { self.capture("alt 1") }
    454               | ^('+' . .)   { self.capture("alt 2") }
    455               ;
    456             ''')
    457 
    458         # AMBIG upon '+' DOWN INT UP etc.. but so what.
    459 
    460         found = self.execTreeParser(
    461             grammar, 'a',
    462             treeGrammar, 'a',
    463             "a + 2")
    464         self.assertEquals("alt 1", found)
    465 
    466 
    467     def testWildcardPlusLookahead(self):
    468         grammar = textwrap.dedent(
    469             r'''
    470             grammar T;
    471             options {language=Python; output=AST;}
    472             a : ID '+'^ INT;
    473             ID : 'a'..'z'+ ;
    474             INT : '0'..'9'+;
    475             SEMI : ';' ;
    476             PERIOD : '.' ;
    477             WS : (' '|'\n') {$channel=HIDDEN;} ;
    478             ''')
    479 
    480         treeGrammar = textwrap.dedent(
    481             r'''
    482             tree grammar TP;
    483             options {language=Python; tokenVocab=T; ASTLabelType=CommonTree;}
    484             a : ^('+' INT INT ) { self.capture("alt 1") }
    485               | ^('+' .+)   { self.capture("alt 2") }
    486               ;
    487             ''')
    488 
    489         # AMBIG upon '+' DOWN INT UP etc.. but so what.
    490 
    491         found = self.execTreeParser(
    492             grammar, 'a',
    493             treeGrammar, 'a',
    494             "a + 2")
    495         self.assertEquals("alt 2", found)
    496 
    497 
    498 if __name__ == '__main__':
    499     unittest.main()
    500