Home | History | Annotate | Download | only in ast-output 
      1 #!/usr/bin/ruby
      2 # encoding: utf-8
      3 
      4 require 'antlr3/test/functional'
      5 
      6 class TestAutoAST < ANTLR3::Test::Functional
      7   
      8   def parse( grammar, rule, input, expect_errors = false )
      9     @grammar = inline_grammar( grammar )
     10     compile_and_load @grammar
     11     grammar_module = self.class.const_get( @grammar.name )
     12     
     13     grammar_module::Lexer.send( :include, ANTLR3::Test::CollectErrors )
     14     grammar_module::Lexer.send( :include, ANTLR3::Test::CaptureOutput )
     15     grammar_module::Parser.send( :include, ANTLR3::Test::CollectErrors )
     16     grammar_module::Parser.send( :include, ANTLR3::Test::CaptureOutput )
     17     
     18     lexer  = grammar_module::Lexer.new( input )
     19     parser = grammar_module::Parser.new( lexer )
     20     
     21     r = parser.send( rule )
     22     parser.reported_errors.should be_empty unless expect_errors
     23     result = ''
     24     
     25     unless r.nil?
     26       result += r.result if r.respond_to?( :result )
     27       result += r.tree.inspect if r.tree
     28     end
     29     return( expect_errors ? [ result, parser.reported_errors ] : result )
     30   end
     31   
     32   def tree_parse( grammar, tree_grammar, rule, tree_rule, input )
     33     @grammar = inline_grammar( grammar )
     34     @tree_grammar = inline_grammar( tree_grammar )
     35     compile_and_load @grammar
     36     compile_and_load @tree_grammar
     37     
     38     grammar_module = self.class.const_get( @grammar.name )
     39     tree_grammar_module = self.class.const_get( @tree_grammar.name )
     40     
     41     grammar_module::Lexer.send( :include, ANTLR3::Test::CollectErrors )
     42     grammar_module::Lexer.send( :include, ANTLR3::Test::CaptureOutput )
     43     grammar_module::Parser.send( :include, ANTLR3::Test::CollectErrors )
     44     grammar_module::Parser.send( :include, ANTLR3::Test::CaptureOutput )
     45     tree_grammar_module::TreeParser.send( :include, ANTLR3::Test::CollectErrors )
     46     tree_grammar_module::TreeParser.send( :include, ANTLR3::Test::CaptureOutput )
     47     
     48     lexer  = grammar_module::Lexer.new( input )
     49     parser = grammar.module::Parser.new( lexer )
     50     r = parser.send( rule )
     51     nodes = ANTLR3::CommonTreeNodeStream( r.tree )
     52     nodes.token_stream = parser.input
     53     walker = tree_grammar_module::TreeParser.new( nodes )
     54     r = walker.send( tree_rule )
     55     
     56     return( r ? r.tree.inspect : '' )
     57   end
     58   
     59   
     60   example 'flat token list' do
     61     result = parse( <<-'END', :a, 'abc 34' )
     62       grammar TokenList;
     63       options {language=Ruby;output=AST;}
     64       a : ID INT ;
     65       ID : 'a'..'z'+ ;
     66       INT : '0'..'9'+;
     67       WS : (' '|'\n') {$channel=HIDDEN;};
     68     END
     69     result.should == 'abc 34'
     70   end
     71   
     72   example 'token list in a single-alternative subrule' do
     73     result = parse( <<-'END', :a, 'abc 34' )
     74       grammar TokenListInSingleAltBlock;
     75       options {language=Ruby;output=AST;}
     76       a : (ID INT) ;
     77       ID : 'a'..'z'+ ;
     78       INT : '0'..'9'+;
     79       WS : (' '|'\n') {$channel=HIDDEN;} ;
     80     END
     81     result.should == 'abc 34'
     82   end
     83   
     84   example "simple root at the outer level via the `^' operator" do
     85     result = parse( <<-'END', :a, 'abc 34' )
     86       grammar SimpleRootAtOuterLevel;
     87       options {language=Ruby;output=AST;}
     88       a : ID^ INT ;
     89       ID : 'a'..'z'+ ;
     90       INT : '0'..'9'+;
     91       WS : (' '|'\n') {$channel=HIDDEN;} ;
     92     END
     93     result.should == '(abc 34)'
     94   end
     95   
     96   example "outer-level root changing token order from the `^' operator" do
     97     result = parse( <<-'END', :a, '34 abc' )
     98       grammar SimpleRootAtOuterLevelReverse;
     99       options {language=Ruby;output=AST;}
    100       a : INT ID^ ;
    101       ID : 'a'..'z'+ ;
    102       INT : '0'..'9'+;
    103       WS : (' '|'\n') {$channel=HIDDEN;} ;
    104     END
    105     result.should == '(abc 34)'
    106   end
    107   
    108   example "leaving out tokens using the `!' operator" do
    109     result = parse( <<-'END', :a, 'abc 34 dag 4532' )
    110       grammar Bang;
    111       options {language=Ruby;output=AST;}
    112       a : ID INT! ID! INT ;
    113       ID : 'a'..'z'+ ;
    114       INT : '0'..'9'+;
    115       WS : (' '|'\n') {$channel=HIDDEN;} ;
    116     END
    117     
    118     result.should == 'abc 4532'
    119   end
    120   
    121   example "tokens in `(...)?' optional subrule" do
    122     result = parse( <<-'END', :a, 'a 1 b' )
    123       grammar OptionalThenRoot;
    124       options {language=Ruby;output=AST;}
    125       a : ( ID INT )? ID^ ;
    126       ID : 'a'..'z'+ ;
    127       INT : '0'..'9'+;
    128       WS : (' '|'\n') {$channel=HIDDEN;} ;
    129     END
    130     result.should == '(b a 1)'
    131   end
    132   
    133   example "labeled literal-string root token" do
    134     result = parse( <<-'END', :a, 'void foo;' )
    135       grammar LabeledStringRoot;
    136       options {language=Ruby;output=AST;}
    137       a : v='void'^ ID ';' ;
    138       ID : 'a'..'z'+ ;
    139       INT : '0'..'9'+;
    140       WS : (' '|'\n') {$channel=HIDDEN;} ;
    141     END
    142     result.should == '(void foo ;)'
    143   end
    144   
    145   example 'rule with token wildcard' do
    146     result = parse( <<-'END', :a, 'void foo;' )
    147       grammar Wildcard;
    148       options {language=Ruby;output=AST;}
    149       a : v='void'^ . ';' ;
    150       ID : 'a'..'z'+ ;
    151       INT : '0'..'9'+;
    152       WS : (' '|'\n') {$channel=HIDDEN;} ;
    153     END
    154     result.should == '(void foo ;)'
    155   end
    156   
    157   example "token wildcard as root via the `^' operator" do
    158     result = parse( <<-'END', :a, 'void foo;' )
    159       grammar WildcardRoot;
    160       options {language=Ruby;output=AST;}
    161       a : v='void' .^ ';' ;
    162       ID : 'a'..'z'+ ;
    163       INT : '0'..'9'+;
    164       WS : (' '|'\n') {$channel=HIDDEN;} ;
    165     END
    166     result.should == '(foo void ;)'
    167   end
    168   
    169   example "labeled token wildcard as root via the `^' operator" do
    170     result = parse( <<-'END', :a, 'void foo;' )
    171       grammar WildcardRootWithLabel;
    172       options {language=Ruby;output=AST;}
    173       a : v='void' x=.^ ';' ;
    174       ID : 'a'..'z'+ ;
    175       INT : '0'..'9'+;
    176       WS : (' '|'\n') {$channel=HIDDEN;} ;
    177     END
    178     result.should == '(foo void ;)'
    179   end
    180   
    181   
    182   example "token wildcard as root (with list label)" do
    183     result = parse( <<-'END', :a, 'void foo;' )
    184       grammar WildcardRootWithListLabel;
    185       options {language=Ruby;output=AST;}
    186       a : v='void' x=.^ ';' ;
    187       ID : 'a'..'z'+ ;
    188       INT : '0'..'9'+;
    189       WS : (' '|'\n') {$channel=HIDDEN;} ;
    190   
    191     END
    192     result.should == '(foo void ;)'
    193   end
    194   
    195   example "trashed token wildcard" do
    196     result = parse( <<-'END', :a, 'void foo;' )
    197       grammar WildcardBangWithListLabel;
    198       options {language=Ruby;output=AST;}
    199       a : v='void' x=.! ';' ;
    200       ID : 'a'..'z'+ ;
    201       INT : '0'..'9'+;
    202       WS : (' '|'\n') {$channel=HIDDEN;} ;
    203   
    204     END
    205     result.should == 'void ;'
    206   end
    207   
    208   example "multiple occurences of the `^' operator in a list of tokens" do
    209     result = parse( <<-'END', :a, 'a 34 c' )
    210       grammar RootRoot;
    211       options {language=Ruby;output=AST;}
    212       a : ID^ INT^ ID ;
    213       ID : 'a'..'z'+ ;
    214       INT : '0'..'9'+;
    215       WS : (' '|'\n') {$channel=HIDDEN;} ;
    216   
    217     END
    218     result.should == '(34 a c)'
    219   end
    220   
    221   example "another case of multiple occurences of the `^' operator" do
    222     result = parse( <<-'END', :a, 'a 34 c' )
    223       grammar RootRoot2;
    224       options {language=Ruby;output=AST;}
    225       a : ID INT^ ID^ ;
    226       ID : 'a'..'z'+ ;
    227       INT : '0'..'9'+;
    228       WS : (' '|'\n') {$channel=HIDDEN;} ;
    229   
    230     END
    231     result.should == '(c (34 a))'
    232   end
    233   
    234   example "root-hoist using `^' from within a (...)+ block" do
    235     result = parse( <<-'END', :a, 'a 34 * b 9 * c' )
    236       grammar RootThenRootInLoop;
    237       options {language=Ruby;output=AST;}
    238       a : ID^ (INT '*'^ ID)+ ;
    239       ID  : 'a'..'z'+ ;
    240       INT : '0'..'9'+;
    241       WS : (' '|'\n') {$channel=HIDDEN;} ;
    242   
    243     END
    244     result.should == '(* (* (a 34) b 9) c)'
    245   end
    246   
    247   example "nested subrules without any AST ops resulting in a flat list" do
    248     result = parse( <<-'END', :a, 'void a b;' )
    249       grammar NestedSubrule;
    250       options {language=Ruby;output=AST;}
    251       a : 'void' (({
    252       #do nothing
    253       } ID|INT) ID | 'null' ) ';' ;
    254       ID : 'a'..'z'+ ;
    255       INT : '0'..'9'+;
    256       WS : (' '|'\n') {$channel=HIDDEN;} ;
    257   
    258     END
    259     result.should == 'void a b ;'
    260   end
    261   
    262   example "invoking another rule without any AST ops, resulting in a flat list" do
    263     result = parse( <<-'END', :a, 'int a' )
    264       grammar InvokeRule;
    265       options {language=Ruby;output=AST;}
    266       a  : type ID ;
    267       type : {
    268         # do nothing
    269       }'int' | 'float' ;
    270       ID : 'a'..'z'+ ;
    271       INT : '0'..'9'+;
    272       WS : (' '|'\n') {$channel=HIDDEN;} ;
    273   
    274     END
    275     result.should == 'int a'
    276   end
    277   
    278   example "hoisting the results of another rule as root using the `^' operator" do
    279     result = parse( <<-'END', :a, 'int a' )
    280       grammar InvokeRuleAsRoot;
    281       options {language=Ruby;output=AST;}
    282       a  : type^ ID ;
    283       type : {
    284         # do nothing
    285       }'int' | 'float' ;
    286       ID : 'a'..'z'+ ;
    287       INT : '0'..'9'+;
    288       WS : (' '|'\n') {$channel=HIDDEN;} ;
    289   
    290     END
    291     result.should == '(int a)'
    292   end
    293   
    294   example "hoisting another rule's true as root using the `^' operator (with a label)" do
    295     result = parse( <<-'END', :a, 'int a' )
    296       grammar InvokeRuleAsRootWithLabel;
    297       options {language=Ruby;output=AST;}
    298       a  : x=type^ ID ;
    299       type : {
    300         # do nothing
    301       }'int' | 'float' ;
    302       ID : 'a'..'z'+ ;
    303       INT : '0'..'9'+;
    304       WS : (' '|'\n') {$channel=HIDDEN;} ;
    305   
    306     END
    307     result.should == '(int a)'
    308   end
    309   
    310   example "hoisting another rule's result tree as root using the `^' operator (with a list += label)" do
    311     result = parse( <<-'END', :a, 'int a' )
    312       grammar InvokeRuleAsRootWithListLabel;
    313       options {language=Ruby;output=AST;}
    314       a  : x+=type^ ID ;
    315       type : {
    316         # do nothing
    317       }'int' | 'float' ;
    318       ID : 'a'..'z'+ ;
    319       INT : '0'..'9'+;
    320       WS : (' '|'\n') {$channel=HIDDEN;} ;
    321   
    322     END
    323     result.should == '(int a)'
    324   end
    325   
    326   example "root-hoist via `^' within a (...)* loop resulting in a deeply-nested tree" do
    327     result = parse( <<-'END', :a, 'a+b+c+d' )
    328       grammar RuleRootInLoop;
    329       options {language=Ruby;output=AST;}
    330       a : ID ('+'^ ID)* ;
    331       ID : 'a'..'z'+ ;
    332       INT : '0'..'9'+;
    333       WS : (' '|'\n') {$channel=HIDDEN;} ;
    334   
    335     END
    336     result.should == '(+ (+ (+ a b) c) d)'
    337   end
    338   
    339   example "hoisting another rule's result tree as root from within a (...)* loop resulting in a deeply nested tree" do
    340     result = parse( <<-'END', :a, 'a+b+c-d' )
    341       grammar RuleInvocationRuleRootInLoop;
    342       options {language=Ruby;output=AST;}
    343       a : ID (op^ ID)* ;
    344       op : {
    345         # do nothing
    346       }'+' | '-' ;
    347       ID : 'a'..'z'+ ;
    348       INT : '0'..'9'+;
    349       WS : (' '|'\n') {$channel=HIDDEN;} ;
    350   
    351     END
    352     result.should == '(- (+ (+ a b) c) d)'
    353   end
    354   
    355   example "using tail recursion to build deeply-nested expression trees" do
    356     result = parse( <<-'END', :s, '3 exp 4 exp 5' )
    357       grammar TailRecursion;
    358       options {language=Ruby;output=AST;}
    359       s : a ;
    360       a : atom ('exp'^ a)? ;
    361       atom : INT ;
    362       ID : 'a'..'z'+ ;
    363       INT : '0'..'9'+;
    364       WS : (' '|'\n') {$channel=HIDDEN;} ;
    365   
    366     END
    367     result.should == '(exp 3 (exp 4 5))'
    368   end
    369   
    370   example "simple token node from a token type set" do
    371     result = parse( <<-'END', :a, 'abc' )
    372       grammar TokenSet;
    373       options {language=Ruby; output=AST;}
    374       a : ID|INT ;
    375       ID : 'a'..'z'+ ;
    376       INT : '0'..'9'+;
    377       WS : (' '|'\n') {$channel=HIDDEN;} ;
    378     END
    379     result.should == 'abc'
    380   end
    381   
    382   example "hoisting a token-type set token as root with `^'" do
    383     result = parse( <<-'END', :a, '+abc' )
    384       grammar SetRoot;
    385       options {language=Ruby;output=AST;}
    386       a : ('+' | '-')^ ID ;
    387       ID : 'a'..'z'+ ;
    388       INT : '0'..'9'+;
    389       WS : (' '|'\n') {$channel=HIDDEN;} ;
    390   
    391     END
    392     result.should == '(+ abc)'
    393   end
    394   
    395   example "hoisting a token-type set token as root with `^' (with a label)" do
    396     result = parse( <<-'END', :a, '+abc' )
    397       grammar SetRootWithLabel;
    398       options {language=Ruby;output=AST;}
    399       a : (x=('+' | '-'))^ ID ;
    400       ID : 'a'..'z'+ ;
    401       INT : '0'..'9'+;
    402       WS : (' '|'\n') {$channel=HIDDEN;} ;
    403   
    404     END
    405     result.should == '+ abc'
    406   end
    407   
    408   example "hoisting a token-type set token as root from within a (...)* loop" do
    409     result = parse( <<-'END', :a, 'a+b-c' )
    410       grammar SetAsRuleRootInLoop;
    411       options {language=Ruby;output=AST;}
    412       a : ID (('+'|'-')^ ID)* ;
    413       ID : 'a'..'z'+ ;
    414       INT : '0'..'9'+;
    415       WS : (' '|'\n') {$channel=HIDDEN;} ;
    416   
    417     END
    418     result.should == '(- (+ a b) c)'
    419   end
    420   
    421   example "an `~' inverted token-type set element" do
    422     result = parse( <<-'END', :a, '34+2' )
    423       grammar NotSet;
    424       options {language=Ruby;output=AST;}
    425       a : ~ID '+' INT ;
    426       ID : 'a'..'z'+ ;
    427       INT : '0'..'9'+;
    428       WS : (' '|'\n') {$channel=HIDDEN;} ;
    429   
    430     END
    431     result.should == '34 + 2'
    432   end
    433   
    434   example "a `~' inverted token-type set in a rule (with a label)" do
    435     result = parse( <<-'END', :a, '34+2' )
    436       grammar NotSetWithLabel;
    437       options {language=Ruby;output=AST;}
    438       a : x=~ID '+' INT ;
    439       ID : 'a'..'z'+ ;
    440       INT : '0'..'9'+;
    441       WS : (' '|'\n') {$channel=HIDDEN;} ;
    442   
    443     END
    444     result.should == '34 + 2'
    445   end
    446   
    447   example "a `~' inverted token-type set element in a rule (with a list += label)" do
    448     result = parse( <<-'END', :a, '34+2' )
    449       grammar NotSetWithListLabel;
    450       options {language=Ruby;output=AST;}
    451       a : x=~ID '+' INT ;
    452       ID : 'a'..'z'+ ;
    453       INT : '0'..'9'+;
    454       WS : (' '|'\n') {$channel=HIDDEN;} ;
    455   
    456     END
    457     result.should == '34 + 2'
    458   end
    459   
    460   example "a `~' inverted token-type set element hoisted to root via `^'" do
    461     result = parse( <<-'END', :a, '34 55' )
    462       grammar NotSetRoot;
    463       options {language=Ruby;output=AST;}
    464       a : ~'+'^ INT ;
    465       ID : 'a'..'z'+ ;
    466       INT : '0'..'9'+;
    467       WS : (' '|'\n') {$channel=HIDDEN;} ;
    468   
    469     END
    470     result.should == '(34 55)'
    471   end
    472   
    473   example "hoisting a `~' inverted token-type set to root using `^' (with label)" do
    474     result = parse( <<-'END', :a, '34 55' )
    475       grammar NotSetRootWithLabel;
    476       options {language=Ruby;output=AST;}
    477       a   : x=~'+'^ INT ;
    478       ID  : 'a'..'z'+ ;
    479       INT : '0'..'9'+;
    480       WS  : (' '|'\n') {$channel=HIDDEN;} ;
    481     END
    482     result.should == '(34 55)'
    483   end
    484   
    485   example "hoisting a `~' inverted token-type set to root using `^' (with list += label)" do
    486     result = parse( <<-'END', :a, '34 55' )
    487       grammar NotSetRootWithListLabel;
    488       options {language=Ruby;output=AST;}
    489       a : x+=~'+'^ INT ;
    490       ID : 'a'..'z'+ ;
    491       INT : '0'..'9'+;
    492       WS : (' '|'\n') {$channel=HIDDEN;} ;
    493     END
    494     result.should == '(34 55)'
    495   end
    496   
    497   example "hoisting a `~' inverted token-type set to root from within a (...)* loop" do
    498     result = parse( <<-'END', :a, '3+4+5' )
    499       grammar NotSetRuleRootInLoop;
    500       options {language=Ruby;output=AST;}
    501       a : INT (~INT^ INT)* ;
    502       blort : '+' ;
    503       ID : 'a'..'z'+ ;
    504       INT : '0'..'9'+;
    505       WS : (' '|'\n') {$channel=HIDDEN;} ;
    506   
    507     END
    508     result.should == '(+ (+ 3 4) 5)'
    509   end
    510   
    511   example "multiple tokens with the same label in a rule" do
    512     result = parse( <<-'END', :a, 'a b' )
    513       grammar TokenLabelReuse;
    514       options {language=Ruby;output=AST;}
    515       a returns [result] : id=ID id=ID {
    516         $result = "2nd id=\%s," \% $id.text
    517       } ;
    518       ID : 'a'..'z'+ ;
    519       INT : '0'..'9'+;
    520       WS : (' '|'\n') {$channel=HIDDEN;} ;
    521   
    522     END
    523     result.should == '2nd id=b,a b'
    524   end
    525   
    526   example "multiple tokens with the same label in a rule (with a `^' root hoist)" do
    527     result = parse( <<-'END', :a, 'a b' )
    528       grammar TokenLabelReuse2;
    529       options {language=Ruby;output=AST;}
    530       a returns [result]: id=ID id=ID^ {$result = "2nd id=#{$id.text},"} ;
    531       ID : 'a'..'z'+ ;
    532       INT : '0'..'9'+;
    533       WS : (' '|'\n') {$channel=HIDDEN;} ;
    534   
    535     END
    536     result.should == '2nd id=b,(b a)'
    537   end
    538   
    539   example "extra token in a simple declaration" do
    540     result, errors = parse( <<-'END', :decl, 'int 34 x=1;', true )
    541       grammar ExtraTokenInSimpleDecl;
    542       options {language=Ruby;output=AST;}
    543       decl : type^ ID '='! INT ';'! ;
    544       type : 'int' | 'float' ;
    545       ID : 'a'..'z'+ ;
    546       INT : '0'..'9'+;
    547       WS : (' '|'\n') {$channel=HIDDEN;} ;
    548   
    549     END
    550     errors.should == [ "line 1:4 extraneous input \"34\" expecting ID" ]
    551     result.should == '(int x 1)'
    552   end
    553   
    554   example "missing ID in a simple declaration" do
    555     result, errors = parse( <<-'END', :decl, 'int =1;', true )
    556       grammar MissingIDInSimpleDecl;
    557       options {language=Ruby;output=AST;}
    558       tokens {EXPR;}
    559       decl : type^ ID '='! INT ';'! ;
    560       type : 'int' | 'float' ;
    561       ID : 'a'..'z'+ ;
    562       INT : '0'..'9'+;
    563       WS : (' '|'\n') {$channel=HIDDEN;} ;
    564     END
    565     errors.should == [ "line 1:4 missing ID at \"=\"" ]
    566     result.should == '(int <missing ID> 1)'
    567   end
    568   
    569   example "missing token of a token-type set in a simple declaration" do
    570     result, errors = parse( <<-'END', :decl, 'x=1;', true )
    571       grammar MissingSetInSimpleDecl;
    572       options {language=Ruby;output=AST;}
    573       tokens {EXPR;}
    574       decl : type^ ID '='! INT ';'! ;
    575       type : 'int' | 'float' ;
    576       ID : 'a'..'z'+ ;
    577       INT : '0'..'9'+;
    578       WS : (' '|'\n') {$channel=HIDDEN;} ;
    579   
    580     END
    581     errors.should == [ "line 1:0 mismatched input \"x\" expecting set nil" ]
    582     result.should == '(<error: x> x 1)'
    583   end
    584   
    585   example "missing INT token simulated with a `<missing INT>' error node" do
    586     result, errors = parse( <<-'END', :a, 'abc', true )
    587       grammar MissingTokenGivesErrorNode;
    588       options {language=Ruby;output=AST;}
    589       a : ID INT ; // follow is EOF
    590       ID : 'a'..'z'+ ;
    591       INT : '0'..'9'+;
    592       WS : (' '|'\n') {$channel=HIDDEN;} ;
    593   
    594     END
    595     errors.should == [ "line 0:-1 missing INT at \"<EOF>\"" ]
    596     result.should == 'abc <missing INT>'
    597   end
    598   
    599   example "missing token from invoked rule results in error node with a resync attribute" do
    600     result, errors = parse( <<-'END', :a, 'abc', true )
    601       grammar MissingTokenGivesErrorNodeInInvokedRule;
    602       options {language=Ruby;output=AST;}
    603       a : b ;
    604       b : ID INT ; // follow should see EOF
    605       ID : 'a'..'z'+ ;
    606       INT : '0'..'9'+;
    607       WS : (' '|'\n') {$channel=HIDDEN;} ;
    608   
    609     END
    610     errors.should == [ "line 0:-1 mismatched input \"<EOF>\" expecting INT" ]
    611     result.should == '<mismatched token: <EOF>, resync = abc>'
    612   end
    613   
    614   example "extraneous ID token displays error and is ignored in AST output" do
    615     result, errors = parse( <<-'END', :a, 'abc ick 34', true )
    616       grammar ExtraTokenGivesErrorNode;
    617       options {language=Ruby;output=AST;}
    618       a : b c ;
    619       b : ID ;
    620       c : INT ;
    621       ID : 'a'..'z'+ ;
    622       INT : '0'..'9'+;
    623       WS : (' '|'\n') {$channel=HIDDEN;} ;
    624   
    625     END
    626     errors.should == [ "line 1:4 extraneous input \"ick\" expecting INT" ]
    627     result.should == 'abc 34'
    628   end
    629   
    630   example "missing ID token simulated with a `<missing ID>' error node" do
    631     result, errors = parse( <<-'END', :a, '34', true )
    632       grammar MissingFirstTokenGivesErrorNode;
    633       options {language=Ruby;output=AST;}
    634       a : ID INT ;
    635       ID : 'a'..'z'+ ;
    636       INT : '0'..'9'+;
    637       WS : (' '|'\n') {$channel=HIDDEN;} ;
    638   
    639     END
    640     errors.should == [ "line 1:0 missing ID at \"34\"" ]
    641     result.should == '<missing ID> 34'
    642   end
    643   
    644   example "another case where a missing ID token is simulated with a `<missing ID>' error node" do
    645     result, errors = parse( <<-'END', :a, '34', true )
    646       grammar MissingFirstTokenGivesErrorNode2;
    647       options {language=Ruby;output=AST;}
    648       a : b c ;
    649       b : ID ;
    650       c : INT ;
    651       ID : 'a'..'z'+ ;
    652       INT : '0'..'9'+;
    653       WS : (' '|'\n') {$channel=HIDDEN;} ;
    654   
    655     END
    656     errors.should == [ "line 1:0 missing ID at \"34\"" ]
    657     result.should == '<missing ID> 34'
    658   end
    659   
    660   example "no viable alternative for rule is represented as a single `<unexpected: ...>' error node" do
    661     result, errors = parse( <<-'END', :a, '*', true )
    662       grammar NoViableAltGivesErrorNode;
    663       options {language=Ruby;output=AST;}
    664       a : b | c ;
    665       b : ID ;
    666       c : INT ;
    667       ID : 'a'..'z'+ ;
    668       S : '*' ;
    669       INT : '0'..'9'+;
    670       WS : (' '|'\n') {$channel=HIDDEN;} ;
    671     END
    672     errors.should == [ "line 1:0 no viable alternative at input \"*\"" ]
    673     result.should == "<unexpected: 0 S[\"*\"] @ line 1 col 0 (0..0), resync = *>"
    674   end
    675   
    676   example "token with a `+=' list label hoisted to root with `^'" do
    677     result = parse( <<-'END', :a, 'a' )
    678       grammar TokenListLabelRuleRoot;
    679       options {language=Ruby;output=AST;}
    680       a : id+=ID^ ;
    681       ID : 'a'..'z'+ ;
    682       INT : '0'..'9'+;
    683       WS : (' '|'\n') {$channel=HIDDEN;} ;
    684   
    685     END
    686     result.should == 'a'
    687   end
    688   
    689   example "token with a list `+=' label trashed with `!'" do
    690     result = parse( <<-'END', :a, 'a' )
    691       grammar TokenListLabelBang;
    692       options {language=Ruby;output=AST;}
    693       a : id+=ID! ;
    694       ID : 'a'..'z'+ ;
    695       INT : '0'..'9'+;
    696       WS : (' '|'\n') {$channel=HIDDEN;} ;
    697   
    698     END
    699     result.should == ''
    700   end
    701   
    702   example "using list `+=' labels to collect trees of invoked rules" do
    703     result = parse( <<-'END', :a, 'a b' )
    704       grammar RuleListLabel;
    705       options {language=Ruby;output=AST;}
    706       a returns [result]: x+=b x+=b {
    707       t = $x[1]
    708       $result = "2nd x=#{t.inspect},"
    709       };
    710       b : ID;
    711       ID : 'a'..'z'+ ;
    712       INT : '0'..'9'+;
    713       WS : (' '|'\n') {$channel=HIDDEN;} ;
    714   
    715     END
    716     result.should == '2nd x=b,a b'
    717   end
    718   
    719   example "using a list `+=' label to collect the trees of invoked rules within a (...)+ block" do
    720     result = parse( <<-'END', :a, 'a b' )
    721       grammar RuleListLabelRuleRoot;
    722       options {language=Ruby;output=AST;}
    723       a returns [result] : ( x+=b^ )+ {
    724       $result = "x=\%s," \% $x[1].inspect
    725       } ;
    726       b : ID;
    727       ID : 'a'..'z'+ ;
    728       INT : '0'..'9'+;
    729       WS : (' '|'\n') {$channel=HIDDEN;} ;
    730   
    731     END
    732     result.should == 'x=(b a),(b a)'
    733   end
    734   
    735   example "trashing the tree of an invoked rule with `!' while collecting the tree with a list `+=' label" do
    736     result = parse( <<-'END', :a, 'a b' )
    737       grammar RuleListLabelBang;
    738       options {language=Ruby;output=AST;}
    739       a returns [result] : x+=b! x+=b {
    740       $result = "1st x=#{$x[0].inspect},"
    741       } ;
    742       b : ID;
    743       ID : 'a'..'z'+ ;
    744       INT : '0'..'9'+;
    745       WS : (' '|'\n') {$channel=HIDDEN;} ;
    746   
    747     END
    748     result.should == '1st x=a,b'
    749   end
    750   
    751   example "a whole bunch of different elements" do
    752     result = parse( <<-'END', :a, 'a b b c c d' )
    753       grammar ComplicatedMelange;
    754       options {language=Ruby;output=AST;}
    755       a : A b=B b=B c+=C c+=C D {s = $D.text} ;
    756       A : 'a' ;
    757       B : 'b' ;
    758       C : 'c' ;
    759       D : 'd' ;
    760       WS : (' '|'\n') {$channel=HIDDEN;} ;
    761     END
    762     result.should == 'a b b c c d'
    763   end
    764   
    765   example "rule return values in addition to AST output" do
    766     result = parse( <<-'END', :a, 'abc 34' )
    767       grammar ReturnValueWithAST;
    768       options {language=Ruby;output=AST;}
    769       a returns [result] : ID b { $result = $b.i.to_s + "\n" } ;
    770       b returns [i] : INT {$i=$INT.text.to_i};
    771       ID : 'a'..'z'+ ;
    772       INT : '0'..'9'+;
    773       WS : (' '|'\n') {$channel=HIDDEN;} ;
    774   
    775     END
    776     result.should == "34\nabc 34"
    777   end
    778   
    779   example "a (...)+ loop containing a token-type set" do
    780     result = parse( <<-'END', :r, 'abc 34 d' )
    781       grammar SetLoop;
    782       options { language=Ruby;output=AST; }
    783       r : (INT|ID)+ ; 
    784       ID : 'a'..'z' + ;
    785       INT : '0'..'9' +;
    786       WS: (' ' | '\n' | '\t')+ {$channel = HIDDEN;};
    787     
    788     END
    789     result.should == 'abc 34 d'
    790   end
    791   
    792 end
    793