Home | History | Annotate | Download | only in runtime 
      1 /*
      2  [The "BSD license"]
      3  Copyright (c) 2005-2009 Terence Parr
      4  All rights reserved.
      5 
      6  Redistribution and use in source and binary forms, with or without
      7  modification, are permitted provided that the following conditions
      8  are met:
      9  1. Redistributions of source code must retain the above copyright
     10      notice, this list of conditions and the following disclaimer.
     11  2. Redistributions in binary form must reproduce the above copyright
     12      notice, this list of conditions and the following disclaimer in the
     13      documentation and/or other materials provided with the distribution.
     14  3. The name of the author may not be used to endorse or promote products
     15      derived from this software without specific prior written permission.
     16 
     17  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     18  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     19  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     20  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     21  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     22  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     23  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     24  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     25  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     26  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     27  */
     28 
     29 package org.antlr.runtime;
     30 
     31 /** The most common stream of tokens where every token is buffered up
     32  *  and tokens are filtered for a certain channel (the parser will only
     33  *  see these tokens).
     34  *
     35  *  Even though it buffers all of the tokens, this token stream pulls tokens
     36  *  from the tokens source on demand. In other words, until you ask for a
     37  *  token using consume(), LT(), etc. the stream does not pull from the lexer.
     38  *
     39  *  The only difference between this stream and BufferedTokenStream superclass
     40  *  is that this stream knows how to ignore off channel tokens. There may be
     41  *  a performance advantage to using the superclass if you don't pass
     42  *  whitespace and comments etc. to the parser on a hidden channel (i.e.,
     43  *  you set $channel instead of calling skip() in lexer rules.)
     44  *
     45  *  @see org.antlr.runtime.UnbufferedTokenStream
     46  *  @see org.antlr.runtime.BufferedTokenStream
     47  */
     48 public class CommonTokenStream extends BufferedTokenStream {
     49     /** Skip tokens on any channel but this one; this is how we skip whitespace... */
     50     protected int channel = Token.DEFAULT_CHANNEL;
     51 
     52     public CommonTokenStream() { ; }
     53 
     54     public CommonTokenStream(TokenSource tokenSource) {
     55         super(tokenSource);
     56     }
     57 
     58     public CommonTokenStream(TokenSource tokenSource, int channel) {
     59         this(tokenSource);
     60         this.channel = channel;
     61     }
     62 
     63     /** Always leave p on an on-channel token. */
     64     public void consume() {
     65         if ( p == -1 ) setup();
     66         p++;
     67         sync(p);
     68         while ( tokens.get(p).getChannel()!=channel ) {
     69             p++;
     70             sync(p);
     71         }
     72     }
     73 
     74     protected Token LB(int k) {
     75         if ( k==0 || (p-k)<0 ) return null;
     76 
     77         int i = p;
     78         int n = 1;
     79         // find k good tokens looking backwards
     80         while ( n<=k ) {
     81             // skip off-channel tokens
     82             i = skipOffTokenChannelsReverse(i-1);
     83             n++;
     84         }
     85         if ( i<0 ) return null;
     86         return tokens.get(i);
     87     }
     88 
     89     public Token LT(int k) {
     90         //System.out.println("enter LT("+k+")");
     91         if ( p == -1 ) setup();
     92         if ( k == 0 ) return null;
     93         if ( k < 0 ) return LB(-k);
     94         int i = p;
     95         int n = 1; // we know tokens[p] is a good one
     96         // find k good tokens
     97         while ( n<k ) {
     98             // skip off-channel tokens
     99             i = skipOffTokenChannels(i+1);
    100             n++;
    101         }
    102 		if ( i>range ) range = i;
    103         return tokens.get(i);
    104     }
    105 
    106     /** Given a starting index, return the index of the first on-channel
    107      *  token.
    108      */
    109     protected int skipOffTokenChannels(int i) {
    110         sync(i);
    111         while ( tokens.get(i).getChannel()!=channel ) { // also stops at EOF (it's onchannel)
    112             i++;
    113             sync(i);
    114         }
    115         return i;
    116     }
    117 
    118     protected int skipOffTokenChannelsReverse(int i) {
    119         while ( i>=0 && ((Token)tokens.get(i)).getChannel()!=channel ) {
    120             i--;
    121         }
    122         return i;
    123     }
    124 
    125     protected void setup() {
    126         p = 0;
    127         sync(0);
    128         int i = 0;
    129         while ( tokens.get(i).getChannel()!=channel ) {
    130             i++;
    131             sync(i);
    132         }
    133         p = i;
    134     }
    135 
    136 	/** Count EOF just once. */
    137 	public int getNumberOfOnChannelTokens() {
    138 		int n = 0;
    139 		fill();
    140 		for (int i = 0; i < tokens.size(); i++) {
    141 			Token t = tokens.get(i);
    142 			if ( t.getChannel()==channel ) n++;
    143 			if ( t.getType()==Token.EOF ) break;
    144 		}
    145 		return n;
    146 	}
    147 
    148     /** Reset this token stream by setting its token source. */
    149     public void setTokenSource(TokenSource tokenSource) {
    150         super.setTokenSource(tokenSource);
    151         channel = Token.DEFAULT_CHANNEL;
    152     }
    153 }
    154