1 /* 2 * [The "BSD license"] 3 * Copyright (c) 2010 Terence Parr 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 package org.antlr.test; 30 31 import org.antlr.runtime.*; 32 import org.antlr.tool.Grammar; 33 import org.antlr.tool.Interpreter; 34 import org.junit.Test; 35 36 /** This actually tests new (12/4/09) buffered but on-demand fetching stream */ 37 public class TestCommonTokenStream extends BaseTest { 38 @Test public void testFirstToken() throws Exception { 39 Grammar g = new Grammar( 40 "lexer grammar t;\n"+ 41 "ID : 'a'..'z'+;\n" + 42 "INT : '0'..'9'+;\n" + 43 "SEMI : ';';\n" + 44 "ASSIGN : '=';\n" + 45 "PLUS : '+';\n" + 46 "MULT : '*';\n" + 47 "WS : ' '+;\n"); 48 // Tokens: 012345678901234567 49 // Input: x = 3 * 0 + 2 * 0; 50 CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;"); 51 Interpreter lexEngine = new Interpreter(g, input); 52 BufferedTokenStream tokens = new BufferedTokenStream(lexEngine); 53 54 String result = tokens.LT(1).getText(); 55 String expecting = "x"; 56 assertEquals(expecting, result); 57 } 58 59 @Test public void test2ndToken() throws Exception { 60 Grammar g = new Grammar( 61 "lexer grammar t;\n"+ 62 "ID : 'a'..'z'+;\n" + 63 "INT : '0'..'9'+;\n" + 64 "SEMI : ';';\n" + 65 "ASSIGN : '=';\n" + 66 "PLUS : '+';\n" + 67 "MULT : '*';\n" + 68 "WS : ' '+;\n"); 69 // Tokens: 012345678901234567 70 // Input: x = 3 * 0 + 2 * 0; 71 CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;"); 72 Interpreter lexEngine = new Interpreter(g, input); 73 BufferedTokenStream tokens = new BufferedTokenStream(lexEngine); 74 75 String result = tokens.LT(2).getText(); 76 String expecting = " "; 77 assertEquals(expecting, result); 78 } 79 80 @Test public void testCompleteBuffer() throws Exception { 81 Grammar g = new Grammar( 82 "lexer grammar t;\n"+ 83 "ID : 'a'..'z'+;\n" + 84 "INT : '0'..'9'+;\n" + 85 "SEMI : ';';\n" + 86 "ASSIGN : '=';\n" + 87 "PLUS : '+';\n" + 88 "MULT : '*';\n" + 89 "WS : ' '+;\n"); 90 // Tokens: 012345678901234567 91 // Input: x = 3 * 0 + 2 * 0; 92 CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;"); 93 Interpreter lexEngine = new Interpreter(g, input); 94 BufferedTokenStream tokens = new BufferedTokenStream(lexEngine); 95 96 int i = 1; 97 Token t = tokens.LT(i); 98 while ( t.getType()!=Token.EOF ) { 99 i++; 100 t = tokens.LT(i); 101 } 102 tokens.LT(i++); // push it past end 103 tokens.LT(i++); 104 105 String result = tokens.toString(); 106 String expecting = "x = 3 * 0 + 2 * 0;"; 107 assertEquals(expecting, result); 108 } 109 110 @Test public void testCompleteBufferAfterConsuming() throws Exception { 111 Grammar g = new Grammar( 112 "lexer grammar t;\n"+ 113 "ID : 'a'..'z'+;\n" + 114 "INT : '0'..'9'+;\n" + 115 "SEMI : ';';\n" + 116 "ASSIGN : '=';\n" + 117 "PLUS : '+';\n" + 118 "MULT : '*';\n" + 119 "WS : ' '+;\n"); 120 // Tokens: 012345678901234567 121 // Input: x = 3 * 0 + 2 * 0; 122 CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;"); 123 Interpreter lexEngine = new Interpreter(g, input); 124 BufferedTokenStream tokens = new BufferedTokenStream(lexEngine); 125 126 Token t = tokens.LT(1); 127 while ( t.getType()!=Token.EOF ) { 128 tokens.consume(); 129 t = tokens.LT(1); 130 } 131 tokens.consume(); 132 tokens.LT(1); // push it past end 133 tokens.consume(); 134 tokens.LT(1); 135 136 String result = tokens.toString(); 137 String expecting = "x = 3 * 0 + 2 * 0;"; 138 assertEquals(expecting, result); 139 } 140 141 @Test public void testLookback() throws Exception { 142 Grammar g = new Grammar( 143 "lexer grammar t;\n"+ 144 "ID : 'a'..'z'+;\n" + 145 "INT : '0'..'9'+;\n" + 146 "SEMI : ';';\n" + 147 "ASSIGN : '=';\n" + 148 "PLUS : '+';\n" + 149 "MULT : '*';\n" + 150 "WS : ' '+;\n"); 151 // Tokens: 012345678901234567 152 // Input: x = 3 * 0 + 2 * 0; 153 CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;"); 154 Interpreter lexEngine = new Interpreter(g, input); 155 BufferedTokenStream tokens = new BufferedTokenStream(lexEngine); 156 157 tokens.consume(); // get x into buffer 158 Token t = tokens.LT(-1); 159 assertEquals("x", t.getText()); 160 161 tokens.consume(); 162 tokens.consume(); // consume '=' 163 t = tokens.LT(-3); 164 assertEquals("x", t.getText()); 165 t = tokens.LT(-2); 166 assertEquals(" ", t.getText()); 167 t = tokens.LT(-1); 168 assertEquals("=", t.getText()); 169 } 170 171 @Test public void testOffChannel() throws Exception { 172 TokenSource lexer = // simulate input " x =34 ;\n" 173 new TokenSource() { 174 int i = 0; 175 Token[] tokens = { 176 new CommonToken(1," "), 177 new CommonToken(1,"x"), 178 new CommonToken(1," "), 179 new CommonToken(1,"="), 180 new CommonToken(1,"34"), 181 new CommonToken(1," "), 182 new CommonToken(1," "), 183 new CommonToken(1,";"), 184 new CommonToken(1,"\n"), 185 new CommonToken(Token.EOF,"") 186 }; 187 { 188 tokens[0].setChannel(Lexer.HIDDEN); 189 tokens[2].setChannel(Lexer.HIDDEN); 190 tokens[5].setChannel(Lexer.HIDDEN); 191 tokens[6].setChannel(Lexer.HIDDEN); 192 tokens[8].setChannel(Lexer.HIDDEN); 193 } 194 public Token nextToken() { 195 return tokens[i++]; 196 } 197 public String getSourceName() { return "test"; } 198 }; 199 200 CommonTokenStream tokens = new CommonTokenStream(lexer); 201 202 assertEquals("x", tokens.LT(1).getText()); // must skip first off channel token 203 tokens.consume(); 204 assertEquals("=", tokens.LT(1).getText()); 205 assertEquals("x", tokens.LT(-1).getText()); 206 207 tokens.consume(); 208 assertEquals("34", tokens.LT(1).getText()); 209 assertEquals("=", tokens.LT(-1).getText()); 210 211 tokens.consume(); 212 assertEquals(";", tokens.LT(1).getText()); 213 assertEquals("34", tokens.LT(-1).getText()); 214 215 tokens.consume(); 216 assertEquals(Token.EOF, tokens.LA(1)); 217 assertEquals(";", tokens.LT(-1).getText()); 218 219 assertEquals("34", tokens.LT(-2).getText()); 220 assertEquals("=", tokens.LT(-3).getText()); 221 assertEquals("x", tokens.LT(-4).getText()); 222 } 223 }
