1 /* 2 * [The "BSD license"] 3 * Copyright (c) 2010 Terence Parr 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 package org.antlr.test; 29 30 import org.antlr.Tool; 31 import org.antlr.codegen.CodeGenerator; 32 import org.stringtemplate.v4.ST; 33 import org.antlr.tool.Grammar; 34 import org.junit.Test; 35 36 public class TestLexer extends BaseTest { 37 protected boolean debug = false; 38 39 /** Public default constructor used by TestRig */ 40 public TestLexer() { 41 } 42 43 @Test public void testSetText() throws Exception { 44 // this must return A not I to the parser; calling a nonfragment rule 45 // from a nonfragment rule does not set the overall token. 46 String grammar = 47 "grammar P;\n"+ 48 "a : A {System.out.println(input);} ;\n"+ 49 "A : '\\\\' 't' {setText(\"\t\");} ;\n" + 50 "WS : (' '|'\\n') {$channel=HIDDEN;} ;"; 51 String found = execParser("P.g", grammar, "PParser", "PLexer", 52 "a", "\\t", debug); 53 assertEquals("\t\n", found); 54 } 55 56 @Test public void testRefToRuleDoesNotSetTokenNorEmitAnother() throws Exception { 57 // this must return A not I to the parser; calling a nonfragment rule 58 // from a nonfragment rule does not set the overall token. 59 String grammar = 60 "grammar P;\n"+ 61 "a : A EOF {System.out.println(input);} ;\n"+ 62 "A : '-' I ;\n" + 63 "I : '0'..'9'+ ;\n"+ 64 "WS : (' '|'\\n') {$channel=HIDDEN;} ;"; 65 String found = execParser("P.g", grammar, "PParser", "PLexer", 66 "a", "-34", debug); 67 assertEquals("-34\n", found); 68 } 69 70 @Test public void testRefToRuleDoesNotSetChannel() throws Exception { 71 // this must set channel of A to HIDDEN. $channel is local to rule 72 // like $type. 73 String grammar = 74 "grammar P;\n"+ 75 "a : A EOF {System.out.println($A.text+\", channel=\"+$A.channel);} ;\n"+ 76 "A : '-' WS I ;\n" + 77 "I : '0'..'9'+ ;\n"+ 78 "WS : (' '|'\\n') {$channel=HIDDEN;} ;"; 79 String found = execParser("P.g", grammar, "PParser", "PLexer", 80 "a", "- 34", debug); 81 assertEquals("- 34, channel=0\n", found); 82 } 83 84 @Test public void testWeCanSetType() throws Exception { 85 String grammar = 86 "grammar P;\n"+ 87 "tokens {X;}\n" + 88 "a : X EOF {System.out.println(input);} ;\n"+ 89 "A : '-' I {$type = X;} ;\n" + 90 "I : '0'..'9'+ ;\n"+ 91 "WS : (' '|'\\n') {$channel=HIDDEN;} ;"; 92 String found = execParser("P.g", grammar, "PParser", "PLexer", 93 "a", "-34", debug); 94 assertEquals("-34\n", found); 95 } 96 97 @Test public void testRefToFragment() throws Exception { 98 // this must return A not I to the parser; calling a nonfragment rule 99 // from a nonfragment rule does not set the overall token. 100 String grammar = 101 "grammar P;\n"+ 102 "a : A {System.out.println(input);} ;\n"+ 103 "A : '-' I ;\n" + 104 "fragment I : '0'..'9'+ ;\n"+ 105 "WS : (' '|'\\n') {$channel=HIDDEN;} ;"; 106 String found = execParser("P.g", grammar, "PParser", "PLexer", 107 "a", "-34", debug); 108 assertEquals("-34\n", found); 109 } 110 111 @Test public void testMultipleRefToFragment() throws Exception { 112 // this must return A not I to the parser; calling a nonfragment rule 113 // from a nonfragment rule does not set the overall token. 114 String grammar = 115 "grammar P;\n"+ 116 "a : A EOF {System.out.println(input);} ;\n"+ 117 "A : I '.' I ;\n" + 118 "fragment I : '0'..'9'+ ;\n"+ 119 "WS : (' '|'\\n') {$channel=HIDDEN;} ;"; 120 String found = execParser("P.g", grammar, "PParser", "PLexer", 121 "a", "3.14159", debug); 122 assertEquals("3.14159\n", found); 123 } 124 125 @Test public void testLabelInSubrule() throws Exception { 126 // can we see v outside? 127 String grammar = 128 "grammar P;\n"+ 129 "a : A EOF ;\n"+ 130 "A : 'hi' WS (v=I)? {$channel=0; System.out.println($v.text);} ;\n" + 131 "fragment I : '0'..'9'+ ;\n"+ 132 "WS : (' '|'\\n') {$channel=HIDDEN;} ;"; 133 String found = execParser("P.g", grammar, "PParser", "PLexer", 134 "a", "hi 342", debug); 135 assertEquals("342\n", found); 136 } 137 138 @Test public void testRefToTokenInLexer() throws Exception { 139 String grammar = 140 "grammar P;\n"+ 141 "a : A EOF ;\n"+ 142 "A : I {System.out.println($I.text);} ;\n" + 143 "fragment I : '0'..'9'+ ;\n"+ 144 "WS : (' '|'\\n') {$channel=HIDDEN;} ;"; 145 String found = execParser("P.g", grammar, "PParser", "PLexer", 146 "a", "342", debug); 147 assertEquals("342\n", found); 148 } 149 150 @Test public void testListLabelInLexer() throws Exception { 151 String grammar = 152 "grammar P;\n"+ 153 "a : A ;\n"+ 154 "A : i+=I+ {for (Object t : $i) System.out.print(\" \"+((Token)t).getText());} ;\n" + 155 "fragment I : '0'..'9'+ ;\n"+ 156 "WS : (' '|'\\n') {$channel=HIDDEN;} ;"; 157 String found = execParser("P.g", grammar, "PParser", "PLexer", 158 "a", "33 297", debug); 159 assertEquals(" 33 297\n", found); 160 } 161 162 @Test public void testDupListRefInLexer() throws Exception { 163 String grammar = 164 "grammar P;\n"+ 165 "a : A ;\n"+ 166 "A : i+=I WS i+=I {$channel=0; for (Object t : $i) System.out.print(\" \"+((Token)t).getText());} ;\n" + 167 "fragment I : '0'..'9'+ ;\n"+ 168 "WS : (' '|'\\n') {$channel=HIDDEN;} ;"; 169 String found = execParser("P.g", grammar, "PParser", "PLexer", 170 "a", "33 297", debug); 171 assertEquals(" 33 297\n", found); 172 } 173 174 @Test public void testCharLabelInLexer() { 175 String grammar = 176 "grammar T;\n" + 177 "a : B ;\n" + 178 "B : x='a' {System.out.println((char)$x);} ;\n" ; 179 String found = execParser("T.g", grammar, "TParser", "TLexer", 180 "a", "a", debug); 181 assertEquals("a\n", found); 182 } 183 184 @Test public void testRepeatedLabelInLexer() { 185 String grammar = 186 "lexer grammar T;\n" + 187 "B : x='a' x='b' ;\n" ; 188 boolean found = 189 rawGenerateAndBuildRecognizer( 190 "T.g", grammar, null, "T", false); 191 boolean expecting = true; // should be ok 192 assertEquals(expecting, found); 193 } 194 195 @Test public void testRepeatedRuleLabelInLexer() { 196 String grammar = 197 "lexer grammar T;\n" + 198 "B : x=A x=A ;\n" + 199 "fragment A : 'a' ;\n" ; 200 boolean found = 201 rawGenerateAndBuildRecognizer( 202 "T.g", grammar, null, "T", false); 203 boolean expecting = true; // should be ok 204 assertEquals(expecting, found); 205 } 206 207 @Test public void testIsolatedEOTEdge() { 208 String grammar = 209 "lexer grammar T;\n" + 210 "QUOTED_CONTENT \n" + 211 " : 'q' (~'q')* (('x' 'q') )* 'q' ; \n"; 212 boolean found = 213 rawGenerateAndBuildRecognizer( 214 "T.g", grammar, null, "T", false); 215 boolean expecting = true; // should be ok 216 assertEquals(expecting, found); 217 } 218 219 @Test public void testEscapedLiterals() { 220 /* Grammar: 221 A : '\"' ; should match a single double-quote: " 222 B : '\\\"' ; should match input \" 223 */ 224 String grammar = 225 "lexer grammar T;\n" + 226 "A : '\\\"' ;\n" + 227 "B : '\\\\\\\"' ;\n" ; // '\\\"' 228 boolean found = 229 rawGenerateAndBuildRecognizer( 230 "T.g", grammar, null, "T", false); 231 boolean expecting = true; // should be ok 232 assertEquals(expecting, found); 233 } 234 235 @Test public void testNewlineLiterals() throws Exception { 236 Grammar g = new Grammar( 237 "lexer grammar T;\n" + 238 "A : '\\n\\n' ;\n" // ANTLR sees '\n\n' 239 ); 240 String expecting = "match(\"\\n\\n\")"; 241 242 Tool antlr = newTool(); 243 antlr.setOutputDirectory(null); // write to /dev/null 244 CodeGenerator generator = new CodeGenerator(antlr, g, "Java"); 245 g.setCodeGenerator(generator); 246 generator.genRecognizer(); // codegen phase sets some vars we need 247 ST codeST = generator.getRecognizerST(); 248 String code = codeST.render(); 249 int m = code.indexOf("match(\""); 250 String found = code.substring(m,m+expecting.length()); 251 252 assertEquals(expecting, found); 253 } 254 } 255
