1 import unittest 2 import textwrap 3 import antlr3 4 import antlr3.tree 5 import testbase 6 import sys 7 8 class TestAutoAST(testbase.ANTLRTest): 9 def parserClass(self, base): 10 class TParser(base): 11 def __init__(self, *args, **kwargs): 12 base.__init__(self, *args, **kwargs) 13 14 self._errors = [] 15 self._output = "" 16 17 18 def capture(self, t): 19 self._output += t 20 21 22 def traceIn(self, ruleName, ruleIndex): 23 self.traces.append('>'+ruleName) 24 25 26 def traceOut(self, ruleName, ruleIndex): 27 self.traces.append('<'+ruleName) 28 29 30 def emitErrorMessage(self, msg): 31 self._errors.append(msg) 32 33 34 return TParser 35 36 37 def lexerClass(self, base): 38 class TLexer(base): 39 def __init__(self, *args, **kwargs): 40 base.__init__(self, *args, **kwargs) 41 42 self._output = "" 43 44 45 def capture(self, t): 46 self._output += t 47 48 49 def traceIn(self, ruleName, ruleIndex): 50 self.traces.append('>'+ruleName) 51 52 53 def traceOut(self, ruleName, ruleIndex): 54 self.traces.append('<'+ruleName) 55 56 57 def recover(self, input, re): 58 # no error recovery yet, just crash! 59 raise 60 61 return TLexer 62 63 64 def execParser(self, grammar, grammarEntry, input, expectErrors=False): 65 lexerCls, parserCls = self.compileInlineGrammar(grammar) 66 67 cStream = antlr3.StringStream(input) 68 lexer = lexerCls(cStream) 69 tStream = antlr3.CommonTokenStream(lexer) 70 parser = parserCls(tStream) 71 r = getattr(parser, grammarEntry)() 72 73 if not expectErrors: 74 self.assertEquals(len(parser._errors), 0, parser._errors) 75 76 result = "" 77 78 if r is not None: 79 if hasattr(r, 'result'): 80 result += r.result 81 82 if r.tree is not None: 83 result += r.tree.toStringTree() 84 85 if not expectErrors: 86 return result 87 88 else: 89 return result, parser._errors 90 91 92 def execTreeParser(self, grammar, grammarEntry, treeGrammar, treeEntry, input): 93 lexerCls, parserCls = self.compileInlineGrammar(grammar) 94 walkerCls = self.compileInlineGrammar(treeGrammar) 95 96 cStream = antlr3.StringStream(input) 97 lexer = lexerCls(cStream) 98 tStream = antlr3.CommonTokenStream(lexer) 99 parser = parserCls(tStream) 100 r = getattr(parser, grammarEntry)() 101 nodes = antlr3.tree.CommonTreeNodeStream(r.tree) 102 nodes.setTokenStream(tStream) 103 walker = walkerCls(nodes) 104 r = getattr(walker, treeEntry)() 105 106 if r is not None: 107 return r.tree.toStringTree() 108 109 return "" 110 111 112 def testTokenList(self): 113 grammar = textwrap.dedent( 114 r''' 115 grammar foo; 116 options {language=Python;output=AST;} 117 a : ID INT ; 118 ID : 'a'..'z'+ ; 119 INT : '0'..'9'+; 120 WS : (' '|'\n') {$channel=HIDDEN;}; 121 ''') 122 123 found = self.execParser(grammar, "a", "abc 34") 124 self.assertEquals("abc 34", found); 125 126 127 def testTokenListInSingleAltBlock(self): 128 grammar = textwrap.dedent( 129 r''' 130 grammar foo; 131 options {language=Python;output=AST;} 132 a : (ID INT) ; 133 ID : 'a'..'z'+ ; 134 INT : '0'..'9'+; 135 WS : (' '|'\n') {$channel=HIDDEN;} ; 136 ''') 137 138 found = self.execParser(grammar,"a", "abc 34") 139 self.assertEquals("abc 34", found) 140 141 142 def testSimpleRootAtOuterLevel(self): 143 grammar = textwrap.dedent( 144 r''' 145 grammar foo; 146 options {language=Python;output=AST;} 147 a : ID^ INT ; 148 ID : 'a'..'z'+ ; 149 INT : '0'..'9'+; 150 WS : (' '|'\n') {$channel=HIDDEN;} ; 151 ''') 152 153 found = self.execParser(grammar, "a", "abc 34") 154 self.assertEquals("(abc 34)", found) 155 156 157 def testSimpleRootAtOuterLevelReverse(self): 158 grammar = textwrap.dedent( 159 r''' 160 grammar T; 161 options {language=Python;output=AST;} 162 a : INT ID^ ; 163 ID : 'a'..'z'+ ; 164 INT : '0'..'9'+; 165 WS : (' '|'\n') {$channel=HIDDEN;} ; 166 ''') 167 168 found = self.execParser(grammar, "a", "34 abc") 169 self.assertEquals("(abc 34)", found) 170 171 172 def testBang(self): 173 grammar = textwrap.dedent( 174 r''' 175 grammar T; 176 options {language=Python;output=AST;} 177 a : ID INT! ID! INT ; 178 ID : 'a'..'z'+ ; 179 INT : '0'..'9'+; 180 WS : (' '|'\n') {$channel=HIDDEN;} ; 181 ''') 182 183 found = self.execParser(grammar, "a", "abc 34 dag 4532") 184 self.assertEquals("abc 4532", found) 185 186 187 def testOptionalThenRoot(self): 188 grammar = textwrap.dedent( 189 r''' 190 grammar T; 191 options {language=Python;output=AST;} 192 a : ( ID INT )? ID^ ; 193 ID : 'a'..'z'+ ; 194 INT : '0'..'9'+; 195 WS : (' '|'\n') {$channel=HIDDEN;} ; 196 ''') 197 198 found = self.execParser(grammar, "a", "a 1 b") 199 self.assertEquals("(b a 1)", found) 200 201 202 def testLabeledStringRoot(self): 203 grammar = textwrap.dedent( 204 r''' 205 grammar T; 206 options {language=Python;output=AST;} 207 a : v='void'^ ID ';' ; 208 ID : 'a'..'z'+ ; 209 INT : '0'..'9'+; 210 WS : (' '|'\n') {$channel=HIDDEN;} ; 211 ''') 212 213 found = self.execParser(grammar, "a", "void foo;") 214 self.assertEquals("(void foo ;)", found) 215 216 217 def testWildcard(self): 218 grammar = textwrap.dedent( 219 r''' 220 grammar T; 221 options {language=Python;output=AST;} 222 a : v='void'^ . ';' ; 223 ID : 'a'..'z'+ ; 224 INT : '0'..'9'+; 225 WS : (' '|'\n') {$channel=HIDDEN;} ; 226 ''') 227 228 found = self.execParser(grammar, "a", "void foo;") 229 self.assertEquals("(void foo ;)", found) 230 231 232 def testWildcardRoot(self): 233 grammar = textwrap.dedent( 234 r''' 235 grammar T; 236 options {language=Python;output=AST;} 237 a : v='void' .^ ';' ; 238 ID : 'a'..'z'+ ; 239 INT : '0'..'9'+; 240 WS : (' '|'\n') {$channel=HIDDEN;} ; 241 ''') 242 243 found = self.execParser(grammar, "a", "void foo;") 244 self.assertEquals("(foo void ;)", found) 245 246 247 def testWildcardRootWithLabel(self): 248 grammar = textwrap.dedent( 249 r''' 250 grammar T; 251 options {language=Python;output=AST;} 252 a : v='void' x=.^ ';' ; 253 ID : 'a'..'z'+ ; 254 INT : '0'..'9'+; 255 WS : (' '|'\n') {$channel=HIDDEN;} ; 256 ''') 257 258 found = self.execParser(grammar, "a", "void foo;") 259 self.assertEquals("(foo void ;)", found) 260 261 262 def testWildcardRootWithListLabel(self): 263 grammar = textwrap.dedent( 264 r''' 265 grammar T; 266 options {language=Python;output=AST;} 267 a : v='void' x=.^ ';' ; 268 ID : 'a'..'z'+ ; 269 INT : '0'..'9'+; 270 WS : (' '|'\n') {$channel=HIDDEN;} ; 271 ''') 272 273 found = self.execParser(grammar, "a", "void foo;") 274 self.assertEquals("(foo void ;)", found) 275 276 277 def testWildcardBangWithListLabel(self): 278 grammar = textwrap.dedent( 279 r''' 280 grammar T; 281 options {language=Python;output=AST;} 282 a : v='void' x=.! ';' ; 283 ID : 'a'..'z'+ ; 284 INT : '0'..'9'+; 285 WS : (' '|'\n') {$channel=HIDDEN;} ; 286 ''') 287 288 found = self.execParser(grammar, "a", "void foo;") 289 self.assertEquals("void ;", found) 290 291 292 def testRootRoot(self): 293 grammar = textwrap.dedent( 294 r''' 295 grammar T; 296 options {language=Python;output=AST;} 297 a : ID^ INT^ ID ; 298 ID : 'a'..'z'+ ; 299 INT : '0'..'9'+; 300 WS : (' '|'\n') {$channel=HIDDEN;} ; 301 ''') 302 303 found = self.execParser(grammar, "a", "a 34 c") 304 self.assertEquals("(34 a c)", found) 305 306 307 def testRootRoot2(self): 308 grammar = textwrap.dedent( 309 r''' 310 grammar T; 311 options {language=Python;output=AST;} 312 a : ID INT^ ID^ ; 313 ID : 'a'..'z'+ ; 314 INT : '0'..'9'+; 315 WS : (' '|'\n') {$channel=HIDDEN;} ; 316 ''') 317 318 found = self.execParser(grammar, "a", "a 34 c") 319 self.assertEquals("(c (34 a))", found) 320 321 322 def testRootThenRootInLoop(self): 323 grammar = textwrap.dedent( 324 r''' 325 grammar T; 326 options {language=Python;output=AST;} 327 a : ID^ (INT '*'^ ID)+ ; 328 ID : 'a'..'z'+ ; 329 INT : '0'..'9'+; 330 WS : (' '|'\n') {$channel=HIDDEN;} ; 331 ''') 332 333 found = self.execParser(grammar, "a", "a 34 * b 9 * c") 334 self.assertEquals("(* (* (a 34) b 9) c)", found) 335 336 337 def testNestedSubrule(self): 338 grammar = textwrap.dedent( 339 r''' 340 grammar T; 341 options {language=Python;output=AST;} 342 a : 'void' (({pass}ID|INT) ID | 'null' ) ';' ; 343 ID : 'a'..'z'+ ; 344 INT : '0'..'9'+; 345 WS : (' '|'\n') {$channel=HIDDEN;} ; 346 ''') 347 348 found = self.execParser(grammar, "a", "void a b;") 349 self.assertEquals("void a b ;", found) 350 351 352 def testInvokeRule(self): 353 grammar = textwrap.dedent( 354 r''' 355 grammar T; 356 options {language=Python;output=AST;} 357 a : type ID ; 358 type : {pass}'int' | 'float' ; 359 ID : 'a'..'z'+ ; 360 INT : '0'..'9'+; 361 WS : (' '|'\n') {$channel=HIDDEN;} ; 362 ''') 363 364 found = self.execParser(grammar, "a", "int a") 365 self.assertEquals("int a", found) 366 367 368 def testInvokeRuleAsRoot(self): 369 grammar = textwrap.dedent( 370 r''' 371 grammar T; 372 options {language=Python;output=AST;} 373 a : type^ ID ; 374 type : {pass}'int' | 'float' ; 375 ID : 'a'..'z'+ ; 376 INT : '0'..'9'+; 377 WS : (' '|'\n') {$channel=HIDDEN;} ; 378 ''') 379 380 found = self.execParser(grammar, "a", "int a") 381 self.assertEquals("(int a)", found) 382 383 384 def testInvokeRuleAsRootWithLabel(self): 385 grammar = textwrap.dedent( 386 r''' 387 grammar T; 388 options {language=Python;output=AST;} 389 a : x=type^ ID ; 390 type : {pass}'int' | 'float' ; 391 ID : 'a'..'z'+ ; 392 INT : '0'..'9'+; 393 WS : (' '|'\n') {$channel=HIDDEN;} ; 394 ''') 395 396 found = self.execParser(grammar, "a", "int a") 397 self.assertEquals("(int a)", found) 398 399 400 def testInvokeRuleAsRootWithListLabel(self): 401 grammar = textwrap.dedent( 402 r''' 403 grammar T; 404 options {language=Python;output=AST;} 405 a : x+=type^ ID ; 406 type : {pass}'int' | 'float' ; 407 ID : 'a'..'z'+ ; 408 INT : '0'..'9'+; 409 WS : (' '|'\n') {$channel=HIDDEN;} ; 410 ''') 411 412 found = self.execParser(grammar, "a", "int a") 413 self.assertEquals("(int a)", found) 414 415 416 def testRuleRootInLoop(self): 417 grammar = textwrap.dedent( 418 r''' 419 grammar T; 420 options {language=Python;output=AST;} 421 a : ID ('+'^ ID)* ; 422 ID : 'a'..'z'+ ; 423 INT : '0'..'9'+; 424 WS : (' '|'\n') {$channel=HIDDEN;} ; 425 ''') 426 427 found = self.execParser(grammar, "a", "a+b+c+d") 428 self.assertEquals("(+ (+ (+ a b) c) d)", found) 429 430 431 def testRuleInvocationRuleRootInLoop(self): 432 grammar = textwrap.dedent( 433 r''' 434 grammar T; 435 options {language=Python;output=AST;} 436 a : ID (op^ ID)* ; 437 op : {pass}'+' | '-' ; 438 ID : 'a'..'z'+ ; 439 INT : '0'..'9'+; 440 WS : (' '|'\n') {$channel=HIDDEN;} ; 441 ''') 442 443 found = self.execParser(grammar, "a", "a+b+c-d") 444 self.assertEquals("(- (+ (+ a b) c) d)", found) 445 446 447 def testTailRecursion(self): 448 grammar = textwrap.dedent( 449 r''' 450 grammar T; 451 options {language=Python;output=AST;} 452 s : a ; 453 a : atom ('exp'^ a)? ; 454 atom : INT ; 455 ID : 'a'..'z'+ ; 456 INT : '0'..'9'+; 457 WS : (' '|'\n') {$channel=HIDDEN;} ; 458 ''') 459 460 found = self.execParser(grammar, "s", "3 exp 4 exp 5") 461 self.assertEquals("(exp 3 (exp 4 5))", found) 462 463 464 def testSet(self): 465 grammar = textwrap.dedent( 466 r''' 467 grammar T; 468 options {language=Python;output=AST;} 469 a : ID|INT ; 470 ID : 'a'..'z'+ ; 471 INT : '0'..'9'+; 472 WS : (' '|'\n') {$channel=HIDDEN;} ; 473 ''') 474 475 found = self.execParser(grammar, "a", "abc") 476 self.assertEquals("abc", found) 477 478 479 def testSetRoot(self): 480 grammar = textwrap.dedent( 481 r''' 482 grammar T; 483 options {language=Python;output=AST;} 484 a : ('+' | '-')^ ID ; 485 ID : 'a'..'z'+ ; 486 INT : '0'..'9'+; 487 WS : (' '|'\n') {$channel=HIDDEN;} ; 488 ''') 489 490 found = self.execParser(grammar, "a", "+abc") 491 self.assertEquals("(+ abc)", found) 492 493 494 @testbase.broken( 495 "FAILS until antlr.g rebuilt in v3", testbase.GrammarCompileError) 496 def testSetRootWithLabel(self): 497 grammar = textwrap.dedent( 498 r''' 499 grammar T; 500 options {language=Python;output=AST;} 501 a : x=('+' | '-')^ ID ; 502 ID : 'a'..'z'+ ; 503 INT : '0'..'9'+; 504 WS : (' '|'\n') {$channel=HIDDEN;} ; 505 ''') 506 507 found = self.execParser(grammar, "a", "+abc") 508 self.assertEquals("(+ abc)", found) 509 510 511 def testSetAsRuleRootInLoop(self): 512 grammar = textwrap.dedent( 513 r''' 514 grammar T; 515 options {language=Python;output=AST;} 516 a : ID (('+'|'-')^ ID)* ; 517 ID : 'a'..'z'+ ; 518 INT : '0'..'9'+; 519 WS : (' '|'\n') {$channel=HIDDEN;} ; 520 ''') 521 522 found = self.execParser(grammar, "a", "a+b-c") 523 self.assertEquals("(- (+ a b) c)", found) 524 525 526 def testNotSet(self): 527 grammar = textwrap.dedent( 528 r''' 529 grammar T; 530 options {language=Python;output=AST;} 531 a : ~ID '+' INT ; 532 ID : 'a'..'z'+ ; 533 INT : '0'..'9'+; 534 WS : (' '|'\n') {$channel=HIDDEN;} ; 535 ''') 536 537 found = self.execParser(grammar, "a", "34+2") 538 self.assertEquals("34 + 2", found) 539 540 541 def testNotSetWithLabel(self): 542 grammar = textwrap.dedent( 543 r''' 544 grammar T; 545 options {language=Python;output=AST;} 546 a : x=~ID '+' INT ; 547 ID : 'a'..'z'+ ; 548 INT : '0'..'9'+; 549 WS : (' '|'\n') {$channel=HIDDEN;} ; 550 ''') 551 552 found = self.execParser(grammar, "a", "34+2") 553 self.assertEquals("34 + 2", found) 554 555 556 def testNotSetWithListLabel(self): 557 grammar = textwrap.dedent( 558 r''' 559 grammar T; 560 options {language=Python;output=AST;} 561 a : x=~ID '+' INT ; 562 ID : 'a'..'z'+ ; 563 INT : '0'..'9'+; 564 WS : (' '|'\n') {$channel=HIDDEN;} ; 565 ''') 566 567 found = self.execParser(grammar, "a", "34+2") 568 self.assertEquals("34 + 2", found) 569 570 571 def testNotSetRoot(self): 572 grammar = textwrap.dedent( 573 r''' 574 grammar T; 575 options {language=Python;output=AST;} 576 a : ~'+'^ INT ; 577 ID : 'a'..'z'+ ; 578 INT : '0'..'9'+; 579 WS : (' '|'\n') {$channel=HIDDEN;} ; 580 ''') 581 582 found = self.execParser(grammar, "a", "34 55") 583 self.assertEquals("(34 55)", found) 584 585 586 def testNotSetRootWithLabel(self): 587 grammar = textwrap.dedent( 588 r''' 589 grammar T; 590 options {language=Python;output=AST;} 591 a : ~'+'^ INT ; 592 ID : 'a'..'z'+ ; 593 INT : '0'..'9'+; 594 WS : (' '|'\n') {$channel=HIDDEN;} ; 595 ''') 596 597 found = self.execParser(grammar, "a", "34 55") 598 self.assertEquals("(34 55)", found) 599 600 601 def testNotSetRootWithListLabel(self): 602 grammar = textwrap.dedent( 603 r''' 604 grammar T; 605 options {language=Python;output=AST;} 606 a : ~'+'^ INT ; 607 ID : 'a'..'z'+ ; 608 INT : '0'..'9'+; 609 WS : (' '|'\n') {$channel=HIDDEN;} ; 610 ''') 611 612 found = self.execParser(grammar, "a", "34 55") 613 self.assertEquals("(34 55)", found) 614 615 616 def testNotSetRuleRootInLoop(self): 617 grammar = textwrap.dedent( 618 r''' 619 grammar T; 620 options {language=Python;output=AST;} 621 a : INT (~INT^ INT)* ; 622 blort : '+' ; 623 ID : 'a'..'z'+ ; 624 INT : '0'..'9'+; 625 WS : (' '|'\n') {$channel=HIDDEN;} ; 626 ''') 627 628 found = self.execParser(grammar, "a", "3+4+5") 629 self.assertEquals("(+ (+ 3 4) 5)", found) 630 631 632 @testbase.broken("FIXME: What happened to the semicolon?", AssertionError) 633 def testTokenLabelReuse(self): 634 # check for compilation problem due to multiple defines 635 grammar = textwrap.dedent( 636 r''' 637 grammar T; 638 options {language=Python;output=AST;} 639 a returns [result] : id=ID id=ID {$result = "2nd id="+$id.text+";";} ; 640 ID : 'a'..'z'+ ; 641 INT : '0'..'9'+; 642 WS : (' '|'\n') {$channel=HIDDEN;} ; 643 ''') 644 645 found = self.execParser(grammar, "a", "a b") 646 self.assertEquals("2nd id=b;a b", found) 647 648 649 def testTokenLabelReuse2(self): 650 # check for compilation problem due to multiple defines 651 grammar = textwrap.dedent( 652 r''' 653 grammar T; 654 options {language=Python;output=AST;} 655 a returns [result]: id=ID id=ID^ {$result = "2nd id="+$id.text+',';} ; 656 ID : 'a'..'z'+ ; 657 INT : '0'..'9'+; 658 WS : (' '|'\n') {$channel=HIDDEN;} ; 659 ''') 660 661 found = self.execParser(grammar, "a", "a b") 662 self.assertEquals("2nd id=b,(b a)", found) 663 664 665 def testTokenListLabelReuse(self): 666 # check for compilation problem due to multiple defines 667 # make sure ids has both ID tokens 668 grammar = textwrap.dedent( 669 r''' 670 grammar T; 671 options {language=Python;output=AST;} 672 a returns [result] : ids+=ID ids+=ID {$result = "id list=["+",".join([t.text for t in $ids])+'],';} ; 673 ID : 'a'..'z'+ ; 674 INT : '0'..'9'+; 675 WS : (' '|'\n') {$channel=HIDDEN;} ; 676 ''') 677 678 found = self.execParser(grammar, "a", "a b") 679 expecting = "id list=[a,b],a b" 680 self.assertEquals(expecting, found) 681 682 683 def testTokenListLabelReuse2(self): 684 # check for compilation problem due to multiple defines 685 # make sure ids has both ID tokens 686 grammar = textwrap.dedent( 687 r''' 688 grammar T; 689 options {language=Python;output=AST;} 690 a returns [result] : ids+=ID^ ids+=ID {$result = "id list=["+",".join([t.text for t in $ids])+'],';} ; 691 ID : 'a'..'z'+ ; 692 INT : '0'..'9'+; 693 WS : (' '|'\n') {$channel=HIDDEN;} ; 694 ''') 695 696 found = self.execParser(grammar, "a", "a b") 697 expecting = "id list=[a,b],(a b)" 698 self.assertEquals(expecting, found) 699 700 701 def testTokenListLabelRuleRoot(self): 702 grammar = textwrap.dedent( 703 r''' 704 grammar T; 705 options {language=Python;output=AST;} 706 a : id+=ID^ ; 707 ID : 'a'..'z'+ ; 708 INT : '0'..'9'+; 709 WS : (' '|'\n') {$channel=HIDDEN;} ; 710 ''') 711 712 found = self.execParser(grammar, "a", "a") 713 self.assertEquals("a", found) 714 715 716 def testTokenListLabelBang(self): 717 grammar = textwrap.dedent( 718 r''' 719 grammar T; 720 options {language=Python;output=AST;} 721 a : id+=ID! ; 722 ID : 'a'..'z'+ ; 723 INT : '0'..'9'+; 724 WS : (' '|'\n') {$channel=HIDDEN;} ; 725 ''') 726 727 found = self.execParser(grammar, "a", "a") 728 self.assertEquals("", found) 729 730 731 def testRuleListLabel(self): 732 grammar = textwrap.dedent( 733 r''' 734 grammar T; 735 options {language=Python;output=AST;} 736 a returns [result]: x+=b x+=b { 737 t=$x[1] 738 $result = "2nd x="+t.toStringTree()+','; 739 }; 740 b : ID; 741 ID : 'a'..'z'+ ; 742 INT : '0'..'9'+; 743 WS : (' '|'\n') {$channel=HIDDEN;} ; 744 ''') 745 746 found = self.execParser(grammar, "a", "a b") 747 self.assertEquals("2nd x=b,a b", found) 748 749 750 def testRuleListLabelRuleRoot(self): 751 grammar = textwrap.dedent( 752 r''' 753 grammar T; 754 options {language=Python;output=AST;} 755 a returns [result] : ( x+=b^ )+ { 756 $result = "x="+$x[1].toStringTree()+','; 757 } ; 758 b : ID; 759 ID : 'a'..'z'+ ; 760 INT : '0'..'9'+; 761 WS : (' '|'\n') {$channel=HIDDEN;} ; 762 ''') 763 764 found = self.execParser(grammar, "a", "a b") 765 self.assertEquals("x=(b a),(b a)", found) 766 767 768 def testRuleListLabelBang(self): 769 grammar = textwrap.dedent( 770 r''' 771 grammar T; 772 options {language=Python;output=AST;} 773 a returns [result] : x+=b! x+=b { 774 $result = "1st x="+$x[0].toStringTree()+','; 775 } ; 776 b : ID; 777 ID : 'a'..'z'+ ; 778 INT : '0'..'9'+; 779 WS : (' '|'\n') {$channel=HIDDEN;} ; 780 ''') 781 782 found = self.execParser(grammar, "a", "a b") 783 self.assertEquals("1st x=a,b", found) 784 785 786 def testComplicatedMelange(self): 787 # check for compilation problem 788 grammar = textwrap.dedent( 789 r''' 790 grammar T; 791 options {language=Python;output=AST;} 792 a : A b=B b=B c+=C c+=C D {s = $D.text} ; 793 A : 'a' ; 794 B : 'b' ; 795 C : 'c' ; 796 D : 'd' ; 797 WS : (' '|'\n') {$channel=HIDDEN;} ; 798 ''') 799 800 found = self.execParser(grammar, "a", "a b b c c d") 801 self.assertEquals("a b b c c d", found) 802 803 804 def testReturnValueWithAST(self): 805 grammar = textwrap.dedent( 806 r''' 807 grammar foo; 808 options {language=Python;output=AST;} 809 a returns [result] : ID b { $result = str($b.i) + '\n';} ; 810 b returns [i] : INT {$i=int($INT.text);} ; 811 ID : 'a'..'z'+ ; 812 INT : '0'..'9'+; 813 WS : (' '|'\n') {$channel=HIDDEN;} ; 814 ''') 815 816 found = self.execParser(grammar, "a", "abc 34") 817 self.assertEquals("34\nabc 34", found) 818 819 820 def testSetLoop(self): 821 grammar = textwrap.dedent( 822 r''' 823 grammar T; 824 options { language=Python;output=AST; } 825 r : (INT|ID)+ ; 826 ID : 'a'..'z' + ; 827 INT : '0'..'9' +; 828 WS: (' ' | '\n' | '\\t')+ {$channel = HIDDEN;}; 829 ''') 830 831 found = self.execParser(grammar, "r", "abc 34 d") 832 self.assertEquals("abc 34 d", found) 833 834 835 def testExtraTokenInSimpleDecl(self): 836 grammar = textwrap.dedent( 837 r''' 838 grammar foo; 839 options {language=Python;output=AST;} 840 decl : type^ ID '='! INT ';'! ; 841 type : 'int' | 'float' ; 842 ID : 'a'..'z'+ ; 843 INT : '0'..'9'+; 844 WS : (' '|'\n') {$channel=HIDDEN;} ; 845 ''') 846 847 found, errors = self.execParser(grammar, "decl", "int 34 x=1;", 848 expectErrors=True) 849 self.assertEquals(["line 1:4 extraneous input u'34' expecting ID"], 850 errors) 851 self.assertEquals("(int x 1)", found) # tree gets correct x and 1 tokens 852 853 854 def testMissingIDInSimpleDecl(self): 855 grammar = textwrap.dedent( 856 r''' 857 grammar foo; 858 options {language=Python;output=AST;} 859 tokens {EXPR;} 860 decl : type^ ID '='! INT ';'! ; 861 type : 'int' | 'float' ; 862 ID : 'a'..'z'+ ; 863 INT : '0'..'9'+; 864 WS : (' '|'\n') {$channel=HIDDEN;} ; 865 ''') 866 867 found, errors = self.execParser(grammar, "decl", "int =1;", 868 expectErrors=True) 869 self.assertEquals(["line 1:4 missing ID at u'='"], errors) 870 self.assertEquals("(int <missing ID> 1)", found) # tree gets invented ID token 871 872 873 def testMissingSetInSimpleDecl(self): 874 grammar = textwrap.dedent( 875 r''' 876 grammar foo; 877 options {language=Python;output=AST;} 878 tokens {EXPR;} 879 decl : type^ ID '='! INT ';'! ; 880 type : 'int' | 'float' ; 881 ID : 'a'..'z'+ ; 882 INT : '0'..'9'+; 883 WS : (' '|'\n') {$channel=HIDDEN;} ; 884 ''') 885 886 found, errors = self.execParser(grammar, "decl", "x=1;", 887 expectErrors=True) 888 self.assertEquals(["line 1:0 mismatched input u'x' expecting set None"], errors) 889 self.assertEquals("(<error: x> x 1)", found) # tree gets invented ID token 890 891 892 def testMissingTokenGivesErrorNode(self): 893 grammar = textwrap.dedent( 894 r''' 895 grammar foo; 896 options {language=Python;output=AST;} 897 a : ID INT ; // follow is EOF 898 ID : 'a'..'z'+ ; 899 INT : '0'..'9'+; 900 WS : (' '|'\n') {$channel=HIDDEN;} ; 901 ''') 902 903 found, errors = self.execParser(grammar, "a", "abc", expectErrors=True) 904 self.assertEquals(["line 1:3 missing INT at '<EOF>'"], errors) 905 self.assertEquals("abc <missing INT>", found) 906 907 908 def testMissingTokenGivesErrorNodeInInvokedRule(self): 909 grammar = textwrap.dedent( 910 r''' 911 grammar foo; 912 options {language=Python;output=AST;} 913 a : b ; 914 b : ID INT ; // follow should see EOF 915 ID : 'a'..'z'+ ; 916 INT : '0'..'9'+; 917 WS : (' '|'\n') {$channel=HIDDEN;} ; 918 ''') 919 920 found, errors = self.execParser(grammar, "a", "abc", expectErrors=True) 921 self.assertEquals(["line 1:3 mismatched input '<EOF>' expecting INT"], errors) 922 self.assertEquals("<mismatched token: <EOF>, resync=abc>", found) 923 924 925 def testExtraTokenGivesErrorNode(self): 926 grammar = textwrap.dedent( 927 r''' 928 grammar foo; 929 options {language=Python;output=AST;} 930 a : b c ; 931 b : ID ; 932 c : INT ; 933 ID : 'a'..'z'+ ; 934 INT : '0'..'9'+; 935 WS : (' '|'\n') {$channel=HIDDEN;} ; 936 ''') 937 938 found, errors = self.execParser(grammar, "a", "abc ick 34", 939 expectErrors=True) 940 self.assertEquals(["line 1:4 extraneous input u'ick' expecting INT"], 941 errors) 942 self.assertEquals("abc 34", found) 943 944 945 def testMissingFirstTokenGivesErrorNode(self): 946 grammar = textwrap.dedent( 947 r''' 948 grammar foo; 949 options {language=Python;output=AST;} 950 a : ID INT ; 951 ID : 'a'..'z'+ ; 952 INT : '0'..'9'+; 953 WS : (' '|'\n') {$channel=HIDDEN;} ; 954 ''') 955 956 found, errors = self.execParser(grammar, "a", "34", expectErrors=True) 957 self.assertEquals(["line 1:0 missing ID at u'34'"], errors) 958 self.assertEquals("<missing ID> 34", found) 959 960 961 def testMissingFirstTokenGivesErrorNode2(self): 962 grammar = textwrap.dedent( 963 r''' 964 grammar foo; 965 options {language=Python;output=AST;} 966 a : b c ; 967 b : ID ; 968 c : INT ; 969 ID : 'a'..'z'+ ; 970 INT : '0'..'9'+; 971 WS : (' '|'\n') {$channel=HIDDEN;} ; 972 ''') 973 974 found, errors = self.execParser(grammar, "a", "34", expectErrors=True) 975 976 # finds an error at the first token, 34, and re-syncs. 977 # re-synchronizing does not consume a token because 34 follows 978 # ref to rule b (start of c). It then matches 34 in c. 979 self.assertEquals(["line 1:0 missing ID at u'34'"], errors) 980 self.assertEquals("<missing ID> 34", found) 981 982 983 def testNoViableAltGivesErrorNode(self): 984 grammar = textwrap.dedent( 985 r''' 986 grammar foo; 987 options {language=Python;output=AST;} 988 a : b | c ; 989 b : ID ; 990 c : INT ; 991 ID : 'a'..'z'+ ; 992 S : '*' ; 993 INT : '0'..'9'+; 994 WS : (' '|'\n') {$channel=HIDDEN;} ; 995 ''') 996 997 found, errors = self.execParser(grammar, "a", "*", expectErrors=True) 998 self.assertEquals(["line 1:0 no viable alternative at input u'*'"], 999 errors) 1000 self.assertEquals("<unexpected: [@0,0:0=u'*',<6>,1:0], resync=*>", 1001 found) 1002 1003 1004 if __name__ == '__main__': 1005 unittest.main() 1006
