1 /* 2 * [The "BSD license"] 3 * Copyright (c) 2010 Terence Parr 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 package org.antlr.test; 29 30 import org.antlr.runtime.ANTLRStringStream; 31 import org.antlr.runtime.CharStream; 32 import org.antlr.runtime.CommonTokenStream; 33 import org.antlr.runtime.Token; 34 import org.antlr.tool.Grammar; 35 import org.antlr.tool.Interpreter; 36 import org.junit.Test; 37 38 public class TestInterpretedLexing extends BaseTest { 39 40 /* 41 static class Tracer implements ANTLRDebugInterface { 42 Grammar g; 43 public DebugActions(Grammar g) { 44 this.g = g; 45 } 46 public void enterRule(String ruleName) { 47 System.out.println("enterRule("+ruleName+")"); 48 } 49 50 public void exitRule(String ruleName) { 51 System.out.println("exitRule("+ruleName+")"); 52 } 53 54 public void matchElement(int type) { 55 System.out.println("matchElement("+g.getTokenName(type)+")"); 56 } 57 58 public void mismatchedElement(MismatchedTokenException e) { 59 System.out.println(e); 60 e.printStackTrace(System.out); 61 } 62 63 public void mismatchedSet(MismatchedSetException e) { 64 System.out.println(e); 65 e.printStackTrace(System.out); 66 } 67 68 public void noViableAlt(NoViableAltException e) { 69 System.out.println(e); 70 e.printStackTrace(System.out); 71 } 72 } 73 */ 74 75 /** Public default constructor used by TestRig */ 76 public TestInterpretedLexing() { 77 } 78 79 @Test public void testSimpleAltCharTest() throws Exception { 80 Grammar g = new Grammar( 81 "lexer grammar t;\n"+ 82 "A : 'a' | 'b' | 'c';"); 83 final int Atype = g.getTokenType("A"); 84 Interpreter engine = new Interpreter(g, new ANTLRStringStream("a")); 85 engine = new Interpreter(g, new ANTLRStringStream("b")); 86 Token result = engine.scan("A"); 87 assertEquals(result.getType(), Atype); 88 engine = new Interpreter(g, new ANTLRStringStream("c")); 89 result = engine.scan("A"); 90 assertEquals(result.getType(), Atype); 91 } 92 93 @Test public void testSingleRuleRef() throws Exception { 94 Grammar g = new Grammar( 95 "lexer grammar t;\n"+ 96 "A : 'a' B 'c' ;\n" + 97 "B : 'b' ;\n"); 98 final int Atype = g.getTokenType("A"); 99 Interpreter engine = new Interpreter(g, new ANTLRStringStream("abc")); // should ignore the x 100 Token result = engine.scan("A"); 101 assertEquals(result.getType(), Atype); 102 } 103 104 @Test public void testSimpleLoop() throws Exception { 105 Grammar g = new Grammar( 106 "lexer grammar t;\n"+ 107 "INT : (DIGIT)+ ;\n"+ 108 "fragment DIGIT : '0'..'9';\n"); 109 final int INTtype = g.getTokenType("INT"); 110 Interpreter engine = new Interpreter(g, new ANTLRStringStream("12x")); // should ignore the x 111 Token result = engine.scan("INT"); 112 assertEquals(result.getType(), INTtype); 113 engine = new Interpreter(g, new ANTLRStringStream("1234")); 114 result = engine.scan("INT"); 115 assertEquals(result.getType(), INTtype); 116 } 117 118 @Test public void testMultAltLoop() throws Exception { 119 Grammar g = new Grammar( 120 "lexer grammar t;\n"+ 121 "A : ('0'..'9'|'a'|'b')+ ;\n"); 122 final int Atype = g.getTokenType("A"); 123 Interpreter engine = new Interpreter(g, new ANTLRStringStream("a")); 124 Token result = engine.scan("A"); 125 engine = new Interpreter(g, new ANTLRStringStream("a")); 126 result = engine.scan("A"); 127 assertEquals(result.getType(), Atype); 128 engine = new Interpreter(g, new ANTLRStringStream("1234")); 129 result = engine.scan("A"); 130 assertEquals(result.getType(), Atype); 131 engine = new Interpreter(g, new ANTLRStringStream("aaa")); 132 result = engine.scan("A"); 133 assertEquals(result.getType(), Atype); 134 engine = new Interpreter(g, new ANTLRStringStream("aaaa9")); 135 result = engine.scan("A"); 136 assertEquals(result.getType(), Atype); 137 engine = new Interpreter(g, new ANTLRStringStream("b")); 138 result = engine.scan("A"); 139 assertEquals(result.getType(), Atype); 140 engine = new Interpreter(g, new ANTLRStringStream("baa")); 141 result = engine.scan("A"); 142 assertEquals(result.getType(), Atype); 143 } 144 145 @Test public void testSimpleLoops() throws Exception { 146 Grammar g = new Grammar( 147 "lexer grammar t;\n"+ 148 "A : ('0'..'9')+ '.' ('0'..'9')* | ('0'..'9')+ ;\n"); 149 final int Atype = g.getTokenType("A"); 150 CharStream input = new ANTLRStringStream("1234.5"); 151 Interpreter engine = new Interpreter(g, input); 152 Token result = engine.scan("A"); 153 assertEquals(Atype, result.getType()); 154 } 155 156 @Test public void testTokensRules() throws Exception { 157 Grammar pg = new Grammar( 158 "parser grammar p;\n"+ 159 "a : (INT|FLOAT|WS)+;\n"); 160 Grammar g = new Grammar(); 161 g.importTokenVocabulary(pg); 162 g.setFileName("<string>"); 163 g.setGrammarContent( 164 "lexer grammar t;\n"+ 165 "INT : (DIGIT)+ ;\n"+ 166 "FLOAT : (DIGIT)+ '.' (DIGIT)* ;\n"+ 167 "fragment DIGIT : '0'..'9';\n" + 168 "WS : (' ')+ {channel=99;};\n"); 169 CharStream input = new ANTLRStringStream("123 139.52"); 170 Interpreter lexEngine = new Interpreter(g, input); 171 172 CommonTokenStream tokens = new CommonTokenStream(lexEngine); 173 tokens.LT(5); // make sure it grabs all tokens 174 String result = tokens.toString(); 175 //System.out.println(result); 176 String expecting = "123 139.52"; 177 assertEquals(expecting, result); 178 } 179 180 } 181
