1 import unittest 2 import textwrap 3 import antlr3 4 import antlr3.tree 5 import testbase 6 import sys 7 8 class T(testbase.ANTLRTest): 9 def parserClass(self, base): 10 class TParser(base): 11 def __init__(self, *args, **kwargs): 12 base.__init__(self, *args, **kwargs) 13 14 self._output = "" 15 16 17 def capture(self, t): 18 self._output += t 19 20 21 def traceIn(self, ruleName, ruleIndex): 22 self.traces.append('>'+ruleName) 23 24 25 def traceOut(self, ruleName, ruleIndex): 26 self.traces.append('<'+ruleName) 27 28 29 def recover(self, input, re): 30 # no error recovery yet, just crash! 31 raise 32 33 return TParser 34 35 36 def lexerClass(self, base): 37 class TLexer(base): 38 def __init__(self, *args, **kwargs): 39 base.__init__(self, *args, **kwargs) 40 41 self._output = "" 42 43 44 def capture(self, t): 45 self._output += t 46 47 48 def traceIn(self, ruleName, ruleIndex): 49 self.traces.append('>'+ruleName) 50 51 52 def traceOut(self, ruleName, ruleIndex): 53 self.traces.append('<'+ruleName) 54 55 56 def recover(self, input, re): 57 # no error recovery yet, just crash! 58 raise 59 60 return TLexer 61 62 63 def execParser(self, grammar, grammarEntry, input): 64 lexerCls, parserCls = self.compileInlineGrammar(grammar) 65 66 cStream = antlr3.StringStream(input) 67 lexer = lexerCls(cStream) 68 tStream = antlr3.CommonTokenStream(lexer) 69 parser = parserCls(tStream) 70 r = getattr(parser, grammarEntry)() 71 72 if r is not None: 73 return r.tree.toStringTree() 74 75 return "" 76 77 78 def execTreeParser(self, grammar, grammarEntry, treeGrammar, treeEntry, input): 79 lexerCls, parserCls = self.compileInlineGrammar(grammar) 80 walkerCls = self.compileInlineGrammar(treeGrammar) 81 82 cStream = antlr3.StringStream(input) 83 lexer = lexerCls(cStream) 84 tStream = antlr3.CommonTokenStream(lexer) 85 parser = parserCls(tStream) 86 r = getattr(parser, grammarEntry)() 87 nodes = antlr3.tree.CommonTreeNodeStream(r.tree) 88 nodes.setTokenStream(tStream) 89 walker = walkerCls(nodes) 90 r = getattr(walker, treeEntry)() 91 92 if r is not None: 93 return r.tree.toStringTree() 94 95 return "" 96 97 98 # PARSERS -- AUTO AST 99 100 def testToken(self): 101 grammar = textwrap.dedent( 102 r''' 103 grammar T1; 104 options { 105 language=Python; 106 output=AST; 107 } 108 @header { 109 class V(CommonTree): 110 def toString(self): 111 return self.token.text + "<V>" 112 __str__ = toString 113 114 } 115 a : ID<V> ; 116 ID : 'a'..'z'+ ; 117 WS : (' '|'\n') {$channel=HIDDEN;} ; 118 ''') 119 120 found = self.execParser( 121 grammar, 'a', 122 input="a" 123 ) 124 125 self.failUnlessEqual("a<V>", found) 126 127 128 def testTokenCommonTree(self): 129 grammar = textwrap.dedent( 130 r''' 131 grammar T; 132 options { 133 language=Python; 134 output=AST; 135 } 136 a : ID<CommonTree> ; 137 ID : 'a'..'z'+ ; 138 WS : (' '|'\n') {$channel=HIDDEN;} ; 139 ''') 140 141 found = self.execParser( 142 grammar, 'a', 143 input="a") 144 145 self.failUnlessEqual("a", found) 146 147 148 def testTokenWithQualifiedType(self): 149 grammar = textwrap.dedent( 150 r''' 151 grammar T; 152 options { 153 language=Python; 154 output=AST; 155 } 156 @members { 157 class V(CommonTree): 158 def toString(self): 159 return self.token.text + "<V>" 160 __str__ = toString 161 } 162 a : ID<TParser.V> ; // TParser.V is qualified name 163 ID : 'a'..'z'+ ; 164 WS : (' '|'\n') {$channel=HIDDEN;} ; 165 ''') 166 167 found = self.execParser( 168 grammar, 'a', 169 input="a" 170 ) 171 172 self.failUnlessEqual("a<V>", found) 173 174 175 def testNamedType(self): 176 grammar = textwrap.dedent( 177 r""" 178 grammar $T; 179 options { 180 language=Python; 181 output=AST; 182 } 183 @header { 184 class V(CommonTree): 185 def toString(self): 186 return self.token.text + "<V>" 187 __str__ = toString 188 } 189 a : ID<node=V> ; 190 ID : 'a'..'z'+ ; 191 WS : (' '|'\\n') {$channel=HIDDEN;} ; 192 """) 193 194 found = self.execParser(grammar, 'a', input="a") 195 self.assertEquals("a<V>", found) 196 197 198 def testTokenWithLabel(self): 199 grammar = textwrap.dedent( 200 r''' 201 grammar T2; 202 options { 203 language=Python; 204 output=AST; 205 } 206 @header { 207 class V(CommonTree): 208 def toString(self): 209 return self.token.text + "<V>" 210 __str__ = toString 211 212 } 213 a : x=ID<V> ; 214 ID : 'a'..'z'+ ; 215 WS : (' '|'\n') {$channel=HIDDEN;} ; 216 ''') 217 218 found = self.execParser( 219 grammar, 'a', 220 input="a" 221 ) 222 223 self.failUnlessEqual("a<V>", found) 224 225 226 def testTokenWithListLabel(self): 227 grammar = textwrap.dedent( 228 r''' 229 grammar T3; 230 options { 231 language=Python; 232 output=AST; 233 } 234 @header { 235 class V(CommonTree): 236 def toString(self): 237 return self.token.text + "<V>" 238 __str__ = toString 239 240 } 241 a : x+=ID<V> ; 242 ID : 'a'..'z'+ ; 243 WS : (' '|'\n') {$channel=HIDDEN;} ; 244 ''') 245 246 found = self.execParser( 247 grammar, 'a', 248 input="a" 249 ) 250 251 self.failUnlessEqual("a<V>", found) 252 253 254 def testTokenRoot(self): 255 grammar = textwrap.dedent( 256 r''' 257 grammar T4; 258 options { 259 language=Python; 260 output=AST; 261 } 262 @header { 263 class V(CommonTree): 264 def toString(self): 265 return self.token.text + "<V>" 266 __str__ = toString 267 268 } 269 a : ID<V>^ ; 270 ID : 'a'..'z'+ ; 271 WS : (' '|'\n') {$channel=HIDDEN;} ; 272 ''') 273 274 found = self.execParser( 275 grammar, 'a', 276 input="a" 277 ) 278 279 self.failUnlessEqual("a<V>", found) 280 281 282 def testTokenRootWithListLabel(self): 283 grammar = textwrap.dedent( 284 r''' 285 grammar T5; 286 options { 287 language=Python; 288 output=AST; 289 } 290 @header { 291 class V(CommonTree): 292 def toString(self): 293 return self.token.text + "<V>" 294 __str__ = toString 295 296 } 297 a : x+=ID<V>^ ; 298 ID : 'a'..'z'+ ; 299 WS : (' '|'\n') {$channel=HIDDEN;} ; 300 ''') 301 302 found = self.execParser( 303 grammar, 'a', 304 input="a" 305 ) 306 307 self.failUnlessEqual("a<V>", found) 308 309 310 def testString(self): 311 grammar = textwrap.dedent( 312 r''' 313 grammar T6; 314 options { 315 language=Python; 316 output=AST; 317 } 318 @header { 319 class V(CommonTree): 320 def toString(self): 321 return self.token.text + "<V>" 322 __str__ = toString 323 324 } 325 a : 'begin'<V> ; 326 ID : 'a'..'z'+ ; 327 WS : (' '|'\n') {$channel=HIDDEN;} ; 328 ''') 329 330 found = self.execParser( 331 grammar, 'a', 332 input="begin" 333 ) 334 335 self.failUnlessEqual("begin<V>", found) 336 337 338 def testStringRoot(self): 339 grammar = textwrap.dedent( 340 r''' 341 grammar T7; 342 options { 343 language=Python; 344 output=AST; 345 } 346 @header { 347 class V(CommonTree): 348 def toString(self): 349 return self.token.text + "<V>" 350 __str__ = toString 351 352 } 353 a : 'begin'<V>^ ; 354 ID : 'a'..'z'+ ; 355 WS : (' '|'\n') {$channel=HIDDEN;} ; 356 ''') 357 358 found = self.execParser( 359 grammar, 'a', 360 input="begin" 361 ) 362 363 self.failUnlessEqual("begin<V>", found) 364 365 366 # PARSERS -- REWRITE AST 367 368 def testRewriteToken(self): 369 grammar = textwrap.dedent( 370 r''' 371 grammar T8; 372 options { 373 language=Python; 374 output=AST; 375 } 376 @header { 377 class V(CommonTree): 378 def toString(self): 379 return self.token.text + "<V>" 380 __str__ = toString 381 382 } 383 a : ID -> ID<V> ; 384 ID : 'a'..'z'+ ; 385 WS : (' '|'\n') {$channel=HIDDEN;} ; 386 ''') 387 388 found = self.execParser( 389 grammar, 'a', 390 input="a" 391 ) 392 393 self.failUnlessEqual("a<V>", found) 394 395 396 def testRewriteTokenWithArgs(self): 397 grammar = textwrap.dedent( 398 r''' 399 grammar T9; 400 options { 401 language=Python; 402 output=AST; 403 } 404 @header { 405 class V(CommonTree): 406 def __init__(self, *args): 407 if len(args) == 4: 408 ttype = args[0] 409 x = args[1] 410 y = args[2] 411 z = args[3] 412 token = CommonToken(type=ttype, text="") 413 414 elif len(args) == 3: 415 ttype = args[0] 416 token = args[1] 417 x = args[2] 418 y, z = 0, 0 419 420 else: 421 raise TypeError("Invalid args \%r" \% (args,)) 422 423 CommonTree.__init__(self, token) 424 self.x = x 425 self.y = y 426 self.z = z 427 428 def toString(self): 429 txt = "" 430 if self.token is not None: 431 txt += self.token.text 432 txt +="<V>;\%d\%d\%d" \% (self.x, self.y, self.z) 433 return txt 434 __str__ = toString 435 436 } 437 a : ID -> ID<V>[42,19,30] ID<V>[$ID,99]; 438 ID : 'a'..'z'+ ; 439 WS : (' '|'\n') {$channel=HIDDEN;} ; 440 ''') 441 442 found = self.execParser( 443 grammar, 'a', 444 input="a" 445 ) 446 447 self.failUnlessEqual("<V>;421930 a<V>;9900", found) 448 449 450 def testRewriteTokenRoot(self): 451 grammar = textwrap.dedent( 452 r''' 453 grammar T10; 454 options { 455 language=Python; 456 output=AST; 457 } 458 @header { 459 class V(CommonTree): 460 def toString(self): 461 return self.token.text + "<V>" 462 __str__ = toString 463 464 } 465 a : ID INT -> ^(ID<V> INT) ; 466 ID : 'a'..'z'+ ; 467 INT : '0'..'9'+ ; 468 WS : (' '|'\n') {$channel=HIDDEN;} ; 469 ''') 470 471 found = self.execParser( 472 grammar, 'a', 473 input="a 2" 474 ) 475 476 self.failUnlessEqual("(a<V> 2)", found) 477 478 479 def testRewriteString(self): 480 grammar = textwrap.dedent( 481 r''' 482 grammar T11; 483 options { 484 language=Python; 485 output=AST; 486 } 487 @header { 488 class V(CommonTree): 489 def toString(self): 490 return self.token.text + "<V>" 491 __str__ = toString 492 493 } 494 a : 'begin' -> 'begin'<V> ; 495 ID : 'a'..'z'+ ; 496 WS : (' '|'\n') {$channel=HIDDEN;} ; 497 ''') 498 499 found = self.execParser( 500 grammar, 'a', 501 input="begin" 502 ) 503 504 self.failUnlessEqual("begin<V>", found) 505 506 507 def testRewriteStringRoot(self): 508 grammar = textwrap.dedent( 509 r''' 510 grammar T12; 511 options { 512 language=Python; 513 output=AST; 514 } 515 @header { 516 class V(CommonTree): 517 def toString(self): 518 return self.token.text + "<V>" 519 __str__ = toString 520 521 } 522 a : 'begin' INT -> ^('begin'<V> INT) ; 523 ID : 'a'..'z'+ ; 524 INT : '0'..'9'+ ; 525 WS : (' '|'\n') {$channel=HIDDEN;} ; 526 ''') 527 528 found = self.execParser( 529 grammar, 'a', 530 input="begin 2" 531 ) 532 533 self.failUnlessEqual("(begin<V> 2)", found) 534 535 def testRewriteRuleResults(self): 536 grammar = textwrap.dedent( 537 r''' 538 grammar T; 539 options { 540 language=Python; 541 output=AST; 542 } 543 tokens {LIST;} 544 @header { 545 class V(CommonTree): 546 def toString(self): 547 return self.token.text + "<V>" 548 __str__ = toString 549 550 class W(CommonTree): 551 def __init__(self, tokenType, txt): 552 super(W, self).__init__( 553 CommonToken(type=tokenType, text=txt)) 554 555 def toString(self): 556 return self.token.text + "<W>" 557 __str__ = toString 558 559 } 560 a : id (',' id)* -> ^(LIST<W>["LIST"] id+); 561 id : ID -> ID<V>; 562 ID : 'a'..'z'+ ; 563 WS : (' '|'\n') {$channel=HIDDEN;} ; 564 ''') 565 566 found = self.execParser( 567 grammar, 'a', 568 input="a,b,c") 569 570 self.failUnlessEqual("(LIST<W> a<V> b<V> c<V>)", found) 571 572 def testCopySemanticsWithHetero(self): 573 grammar = textwrap.dedent( 574 r''' 575 grammar T; 576 options { 577 language=Python; 578 output=AST; 579 } 580 @header { 581 class V(CommonTree): 582 def dupNode(self): 583 return V(self) 584 585 def toString(self): 586 return self.token.text + "<V>" 587 __str__ = toString 588 589 } 590 a : type ID (',' ID)* ';' -> ^(type ID)+; 591 type : 'int'<V> ; 592 ID : 'a'..'z'+ ; 593 INT : '0'..'9'+; 594 WS : (' '|'\\n') {$channel=HIDDEN;} ; 595 ''') 596 597 found = self.execParser( 598 grammar, 'a', 599 input="int a, b, c;") 600 self.failUnlessEqual("(int<V> a) (int<V> b) (int<V> c)", found) 601 602 # TREE PARSERS -- REWRITE AST 603 604 def testTreeParserRewriteFlatList(self): 605 grammar = textwrap.dedent( 606 r''' 607 grammar T13; 608 options { 609 language=Python; 610 output=AST; 611 } 612 a : ID INT; 613 ID : 'a'..'z'+ ; 614 INT : '0'..'9'+; 615 WS : (' '|'\n') {$channel=HIDDEN;} ; 616 ''') 617 618 treeGrammar = textwrap.dedent( 619 r''' 620 tree grammar TP13; 621 options { 622 language=Python; 623 output=AST; 624 ASTLabelType=CommonTree; 625 tokenVocab=T13; 626 } 627 @header { 628 class V(CommonTree): 629 def toString(self): 630 return self.token.text + "<V>" 631 __str__ = toString 632 633 class W(CommonTree): 634 def toString(self): 635 return self.token.text + "<W>" 636 __str__ = toString 637 638 } 639 a : ID INT -> INT<V> ID<W> 640 ; 641 ''') 642 643 found = self.execTreeParser( 644 grammar, 'a', 645 treeGrammar, 'a', 646 input="abc 34" 647 ) 648 649 self.failUnlessEqual("34<V> abc<W>", found) 650 651 652 def testTreeParserRewriteTree(self): 653 grammar = textwrap.dedent( 654 r''' 655 grammar T14; 656 options { 657 language=Python; 658 output=AST; 659 } 660 a : ID INT; 661 ID : 'a'..'z'+ ; 662 INT : '0'..'9'+; 663 WS : (' '|'\n') {$channel=HIDDEN;} ; 664 ''') 665 666 treeGrammar = textwrap.dedent( 667 r''' 668 tree grammar TP14; 669 options { 670 language=Python; 671 output=AST; 672 ASTLabelType=CommonTree; 673 tokenVocab=T14; 674 } 675 @header { 676 class V(CommonTree): 677 def toString(self): 678 return self.token.text + "<V>" 679 __str__ = toString 680 681 class W(CommonTree): 682 def toString(self): 683 return self.token.text + "<W>" 684 __str__ = toString 685 686 } 687 a : ID INT -> ^(INT<V> ID<W>) 688 ; 689 ''') 690 691 found = self.execTreeParser( 692 grammar, 'a', 693 treeGrammar, 'a', 694 input="abc 34" 695 ) 696 697 self.failUnlessEqual("(34<V> abc<W>)", found) 698 699 700 def testTreeParserRewriteImaginary(self): 701 grammar = textwrap.dedent( 702 r''' 703 grammar T15; 704 options { 705 language=Python; 706 output=AST; 707 } 708 a : ID ; 709 ID : 'a'..'z'+ ; 710 INT : '0'..'9'+; 711 WS : (' '|'\n') {$channel=HIDDEN;} ; 712 ''') 713 714 treeGrammar = textwrap.dedent( 715 r''' 716 tree grammar TP15; 717 options { 718 language=Python; 719 output=AST; 720 ASTLabelType=CommonTree; 721 tokenVocab=T15; 722 } 723 tokens { ROOT; } 724 @header { 725 class V(CommonTree): 726 def __init__(self, tokenType): 727 CommonTree.__init__(self, CommonToken(tokenType)) 728 729 def toString(self): 730 return tokenNames[self.token.type] + "<V>" 731 __str__ = toString 732 733 734 } 735 a : ID -> ROOT<V> ID 736 ; 737 ''') 738 739 found = self.execTreeParser( 740 grammar, 'a', 741 treeGrammar, 'a', 742 input="abc" 743 ) 744 745 self.failUnlessEqual("ROOT<V> abc", found) 746 747 748 def testTreeParserRewriteImaginaryWithArgs(self): 749 grammar = textwrap.dedent( 750 r''' 751 grammar T16; 752 options { 753 language=Python; 754 output=AST; 755 } 756 a : ID ; 757 ID : 'a'..'z'+ ; 758 INT : '0'..'9'+; 759 WS : (' '|'\n') {$channel=HIDDEN;} ; 760 ''') 761 762 treeGrammar = textwrap.dedent( 763 r''' 764 tree grammar TP16; 765 options { 766 language=Python; 767 output=AST; 768 ASTLabelType=CommonTree; 769 tokenVocab=T16; 770 } 771 tokens { ROOT; } 772 @header { 773 class V(CommonTree): 774 def __init__(self, tokenType, x): 775 CommonTree.__init__(self, CommonToken(tokenType)) 776 self.x = x 777 778 def toString(self): 779 return tokenNames[self.token.type] + "<V>;" + str(self.x) 780 __str__ = toString 781 782 } 783 a : ID -> ROOT<V>[42] ID 784 ; 785 ''') 786 787 found = self.execTreeParser( 788 grammar, 'a', 789 treeGrammar, 'a', 790 input="abc" 791 ) 792 793 self.failUnlessEqual("ROOT<V>;42 abc", found) 794 795 796 def testTreeParserRewriteImaginaryRoot(self): 797 grammar = textwrap.dedent( 798 r''' 799 grammar T17; 800 options { 801 language=Python; 802 output=AST; 803 } 804 a : ID ; 805 ID : 'a'..'z'+ ; 806 INT : '0'..'9'+; 807 WS : (' '|'\n') {$channel=HIDDEN;} ; 808 ''') 809 810 treeGrammar = textwrap.dedent( 811 r''' 812 tree grammar TP17; 813 options { 814 language=Python; 815 output=AST; 816 ASTLabelType=CommonTree; 817 tokenVocab=T17; 818 } 819 tokens { ROOT; } 820 @header { 821 class V(CommonTree): 822 def __init__(self, tokenType): 823 CommonTree.__init__(self, CommonToken(tokenType)) 824 825 def toString(self): 826 return tokenNames[self.token.type] + "<V>" 827 __str__ = toString 828 829 } 830 a : ID -> ^(ROOT<V> ID) 831 ; 832 ''') 833 834 found = self.execTreeParser( 835 grammar, 'a', 836 treeGrammar, 'a', 837 input="abc" 838 ) 839 840 self.failUnlessEqual("(ROOT<V> abc)", found) 841 842 843 def testTreeParserRewriteImaginaryFromReal(self): 844 grammar = textwrap.dedent( 845 r''' 846 grammar T18; 847 options { 848 language=Python; 849 output=AST; 850 } 851 a : ID ; 852 ID : 'a'..'z'+ ; 853 INT : '0'..'9'+; 854 WS : (' '|'\n') {$channel=HIDDEN;} ; 855 ''') 856 857 treeGrammar = textwrap.dedent( 858 r''' 859 tree grammar TP18; 860 options { 861 language=Python; 862 output=AST; 863 ASTLabelType=CommonTree; 864 tokenVocab=T18; 865 } 866 tokens { ROOT; } 867 @header { 868 class V(CommonTree): 869 def __init__(self, tokenType, tree=None): 870 if tree is None: 871 CommonTree.__init__(self, CommonToken(tokenType)) 872 else: 873 CommonTree.__init__(self, tree) 874 self.token.type = tokenType 875 876 def toString(self): 877 return tokenNames[self.token.type]+"<V>@"+str(self.token.line) 878 __str__ = toString 879 880 } 881 a : ID -> ROOT<V>[$ID] 882 ; 883 ''') 884 885 found = self.execTreeParser( 886 grammar, 'a', 887 treeGrammar, 'a', 888 input="abc" 889 ) 890 891 self.failUnlessEqual("ROOT<V>@1", found) 892 893 894 def testTreeParserAutoHeteroAST(self): 895 grammar = textwrap.dedent( 896 r''' 897 grammar T; 898 options { 899 language=Python; 900 output=AST; 901 } 902 a : ID ';' ; 903 ID : 'a'..'z'+ ; 904 INT : '0'..'9'+; 905 WS : (' '|'\n') {$channel=HIDDEN;} ; 906 ''') 907 908 treeGrammar = textwrap.dedent( 909 r''' 910 tree grammar TP; 911 options { 912 language=Python; 913 output=AST; 914 ASTLabelType=CommonTree; 915 tokenVocab=T; 916 } 917 tokens { ROOT; } 918 @header { 919 class V(CommonTree): 920 def toString(self): 921 return CommonTree.toString(self) + "<V>" 922 __str__ = toString 923 924 } 925 926 a : ID<V> ';'<V>; 927 ''') 928 929 found = self.execTreeParser( 930 grammar, 'a', 931 treeGrammar, 'a', 932 input="abc;" 933 ) 934 935 self.failUnlessEqual("abc<V> ;<V>", found) 936 937 938 if __name__ == '__main__': 939 unittest.main() 940
