Home | History | Annotate | Download | only in Antlr3.Runtime 
      1 /*
      2  * [The "BSD licence"]
      3  * Copyright (c) 2005-2008 Terence Parr
      4  * All rights reserved.
      5  *
      6  * Conversion to C#:
      7  * Copyright (c) 2009 Sam Harwell
      8  * All rights reserved.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  * 3. The name of the author may not be used to endorse or promote products
     19  *    derived from this software without specific prior written permission.
     20  *
     21  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     22  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     23  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     24  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     25  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     26  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     27  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     28  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     29  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     30  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     31  */
     32 
     33 namespace Antlr.Runtime
     34 {
     35     using Antlr.Runtime.Misc;
     36     using CLSCompliant = System.CLSCompliantAttribute;
     37     using NotSupportedException = System.NotSupportedException;
     38     using IndexOutOfRangeException = System.IndexOutOfRangeException;
     39 
     40     /** A token stream that pulls tokens from the code source on-demand and
     41      *  without tracking a complete buffer of the tokens. This stream buffers
     42      *  the minimum number of tokens possible.  It's the same as
     43      *  OnDemandTokenStream except that OnDemandTokenStream buffers all tokens.
     44      *
     45      *  You can't use this stream if you pass whitespace or other off-channel
     46      *  tokens to the parser. The stream can't ignore off-channel tokens.
     47      *
     48      *  You can only look backwards 1 token: LT(-1).
     49      *
     50      *  Use this when you need to read from a socket or other infinite stream.
     51      *
     52      *  @see BufferedTokenStream
     53      *  @see CommonTokenStream
     54      */
     55     public class UnbufferedTokenStream : LookaheadStream<IToken>, ITokenStream, ITokenStreamInformation
     56     {
     57         [CLSCompliant(false)]
     58         protected ITokenSource tokenSource;
     59         protected int tokenIndex; // simple counter to set token index in tokens
     60 
     61         /** Skip tokens on any channel but this one; this is how we skip whitespace... */
     62         protected int channel = TokenChannels.Default;
     63 
     64         private readonly ListStack<IToken> _realTokens = new ListStack<IToken>() { null };
     65 
     66         public UnbufferedTokenStream(ITokenSource tokenSource)
     67         {
     68             this.tokenSource = tokenSource;
     69         }
     70 
     71         public ITokenSource TokenSource
     72         {
     73             get
     74             {
     75                 return this.tokenSource;
     76             }
     77         }
     78 
     79         public string SourceName
     80         {
     81             get
     82             {
     83                 return TokenSource.SourceName;
     84             }
     85         }
     86 
     87         #region ITokenStreamInformation Members
     88 
     89         public IToken LastToken
     90         {
     91             get
     92             {
     93                 return LB(1);
     94             }
     95         }
     96 
     97         public IToken LastRealToken
     98         {
     99             get
    100             {
    101                 return _realTokens.Peek();
    102             }
    103         }
    104 
    105         public int MaxLookBehind
    106         {
    107             get
    108             {
    109                 return 1;
    110             }
    111         }
    112 
    113         public override int Mark()
    114         {
    115             _realTokens.Push(_realTokens.Peek());
    116             return base.Mark();
    117         }
    118 
    119         public override void Release(int marker)
    120         {
    121             base.Release(marker);
    122             _realTokens.Pop();
    123         }
    124 
    125         public override void Clear()
    126         {
    127             _realTokens.Clear();
    128             _realTokens.Push(null);
    129         }
    130 
    131         public override void Consume()
    132         {
    133             base.Consume();
    134             if (PreviousElement != null && PreviousElement.Line > 0)
    135                 _realTokens[_realTokens.Count - 1] = PreviousElement;
    136         }
    137 
    138         #endregion
    139 
    140         public override IToken NextElement()
    141         {
    142             IToken t = this.tokenSource.NextToken();
    143             t.TokenIndex = this.tokenIndex++;
    144             return t;
    145         }
    146 
    147         public override bool IsEndOfFile(IToken o)
    148         {
    149             return o.Type == CharStreamConstants.EndOfFile;
    150         }
    151 
    152         public IToken Get(int i)
    153         {
    154             throw new NotSupportedException("Absolute token indexes are meaningless in an unbuffered stream");
    155         }
    156 
    157         public int LA(int i)
    158         {
    159             return LT(i).Type;
    160         }
    161 
    162         public string ToString(int start, int stop)
    163         {
    164             return "n/a";
    165         }
    166 
    167         public string ToString(IToken start, IToken stop)
    168         {
    169             return "n/a";
    170         }
    171     }
    172 }
    173