1 #!/usr/bin/ruby 2 # encoding: utf-8 3 4 require 'antlr3' 5 6 module ANTLR3 7 =begin rdoc ANTLR3::FilterMode 8 9 If a lexer grammar specifies the <tt>filter = true</t> option, the generated 10 Lexer code will include this module. It modifies the standard 11 <tt>next_token</tt> to catch RecognitionErrors and skip ahead in the input until 12 the token! method can match a token without raising a RecognitionError. 13 14 See http://www.antlr.org/wiki/display/ANTLR3/Lexical+filters for more info on 15 lexer filter mode. 16 17 =end 18 module FilterMode 19 def next_token 20 # if at end-of-file, return the EOF token 21 @input.peek == ANTLR3::EOF and return ANTLR3::EOF_TOKEN 22 23 @state.token = nil 24 @state.channel = ANTLR3::DEFAULT_CHANNEL 25 @state.token_start_position = @input.index 26 @state.token_start_column = @input.column 27 @state.token_start_line = @input.line 28 @state.text = nil 29 @state.backtracking = 1 30 31 m = @input.mark 32 token! 33 @input.release( m ) 34 emit 35 return @state.token 36 rescue ANTLR3::BacktrackingFailed 37 # token! backtracks with synpred at backtracking==2 38 # and we set the synpredgate to allow actions at level 1. 39 @input.rewind( m ) 40 @input.consume # advance one char and try again 41 retry 42 rescue ANTLR3::Error::RecognitionError => re 43 # shouldn't happen in backtracking mode, but... 44 report_error( re ) 45 recover( re ) 46 ensure 47 @state.backtracking = 0 48 end 49 50 def memoize( rule, start_index, success ) 51 super( rule, start_index, success ) if @state.backtracking > 1 52 end 53 54 def already_parsed_rule?( rule ) 55 @state.backtracking > 1 ? super( rule ) : false 56 end 57 end 58 end 59
