1 /* 2 * [The "BSD license"] 3 * Copyright (c) 2010 Terence Parr 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 package org.antlr.codegen; 29 30 31 import org.antlr.Tool; 32 import org.antlr.analysis.DFA; 33 import org.antlr.analysis.*; 34 import org.antlr.grammar.v3.ANTLRLexer; 35 import org.antlr.grammar.v3.ANTLRParser; 36 import org.antlr.grammar.v3.ActionTranslator; 37 import org.antlr.grammar.v3.CodeGenTreeWalker; 38 import org.antlr.misc.BitSet; 39 import org.antlr.misc.*; 40 import org.antlr.runtime.*; 41 import org.antlr.runtime.tree.CommonTreeNodeStream; 42 import org.antlr.tool.*; 43 import org.stringtemplate.v4.*; 44 45 import java.io.IOException; 46 import java.io.Writer; 47 import java.util.*; 48 49 /** ANTLR's code generator. 50 * 51 * Generate recognizers derived from grammars. Language independence 52 * achieved through the use of STGroup objects. All output 53 * strings are completely encapsulated in the group files such as Java.stg. 54 * Some computations are done that are unused by a particular language. 55 * This generator just computes and sets the values into the templates; 56 * the templates are free to use or not use the information. 57 * 58 * To make a new code generation target, define X.stg for language X 59 * by copying from existing Y.stg most closely releated to your language; 60 * e.g., to do CSharp.stg copy Java.stg. The template group file has a 61 * bunch of templates that are needed by the code generator. You can add 62 * a new target w/o even recompiling ANTLR itself. The language=X option 63 * in a grammar file dictates which templates get loaded/used. 64 * 65 * Some language like C need both parser files and header files. Java needs 66 * to have a separate file for the cyclic DFA as ANTLR generates bytecodes 67 * directly (which cannot be in the generated parser Java file). To facilitate 68 * this, 69 * 70 * cyclic can be in same file, but header, output must be searpate. recognizer 71 * is in outptufile. 72 */ 73 public class CodeGenerator { 74 /** When generating SWITCH statements, some targets might need to limit 75 * the size (based upon the number of case labels). Generally, this 76 * limit will be hit only for lexers where wildcard in a UNICODE 77 * vocabulary environment would generate a SWITCH with 65000 labels. 78 */ 79 public final static int MSCL_DEFAULT = 300; 80 public static int MAX_SWITCH_CASE_LABELS = MSCL_DEFAULT; 81 public final static int MSA_DEFAULT = 3; 82 public static int MIN_SWITCH_ALTS = MSA_DEFAULT; 83 public boolean GENERATE_SWITCHES_WHEN_POSSIBLE = true; 84 public static boolean LAUNCH_ST_INSPECTOR = false; 85 public final static int MADSI_DEFAULT = 60; // do lots of states inline (needed for expression rules) 86 public static int MAX_ACYCLIC_DFA_STATES_INLINE = MADSI_DEFAULT; 87 88 public static String classpathTemplateRootDirectoryName = 89 "org/antlr/codegen/templates"; 90 91 /** Which grammar are we generating code for? Each generator 92 * is attached to a specific grammar. 93 */ 94 public Grammar grammar; 95 96 /** What language are we generating? */ 97 protected String language; 98 99 /** The target specifies how to write out files and do other language 100 * specific actions. 101 */ 102 public Target target = null; 103 104 /** Where are the templates this generator should use to generate code? */ 105 protected STGroup templates; 106 107 /** The basic output templates without AST or templates stuff; this will be 108 * the templates loaded for the language such as Java.stg *and* the Dbg 109 * stuff if turned on. This is used for generating syntactic predicates. 110 */ 111 protected STGroup baseTemplates; 112 113 protected ST recognizerST; 114 protected ST outputFileST; 115 protected ST headerFileST; 116 117 /** Used to create unique labels */ 118 protected int uniqueLabelNumber = 1; 119 120 /** A reference to the ANTLR tool so we can learn about output directories 121 * and such. 122 */ 123 protected Tool tool; 124 125 /** Generate debugging event method calls */ 126 protected boolean debug; 127 128 /** Create a Tracer object and make the recognizer invoke this. */ 129 protected boolean trace; 130 131 /** Track runtime parsing information about decisions etc... 132 * This requires the debugging event mechanism to work. 133 */ 134 protected boolean profile; 135 136 protected int lineWidth = 72; 137 138 /** I have factored out the generation of acyclic DFAs to separate class */ 139 public ACyclicDFACodeGenerator acyclicDFAGenerator = 140 new ACyclicDFACodeGenerator(this); 141 142 /** I have factored out the generation of cyclic DFAs to separate class */ 143 /* 144 public CyclicDFACodeGenerator cyclicDFAGenerator = 145 new CyclicDFACodeGenerator(this); 146 */ 147 148 public static final String VOCAB_FILE_EXTENSION = ".tokens"; 149 protected final static String vocabFilePattern = 150 "<tokens:{it|<it.name>=<it.type>\n}>" + 151 "<literals:{it|<it.name>=<it.type>\n}>"; 152 153 public CodeGenerator(Tool tool, Grammar grammar, String language) { 154 this.tool = tool; 155 this.grammar = grammar; 156 this.language = language; 157 target = loadLanguageTarget(language); 158 } 159 160 public static Target loadLanguageTarget(String language) { 161 Target target = null; 162 String targetName = "org.antlr.codegen."+language+"Target"; 163 try { 164 Class<? extends Target> c = Class.forName(targetName).asSubclass(Target.class); 165 target = (Target)c.newInstance(); 166 } 167 catch (ClassNotFoundException cnfe) { 168 target = new Target(); // use default 169 } 170 catch (InstantiationException ie) { 171 ErrorManager.error(ErrorManager.MSG_CANNOT_CREATE_TARGET_GENERATOR, 172 targetName, 173 ie); 174 } 175 catch (IllegalAccessException cnfe) { 176 ErrorManager.error(ErrorManager.MSG_CANNOT_CREATE_TARGET_GENERATOR, 177 targetName, 178 cnfe); 179 } 180 return target; 181 } 182 183 /** load the main language.stg template group file */ 184 public void loadTemplates(String language) { 185 String langDir = classpathTemplateRootDirectoryName+"/"+language; 186 STGroup coreTemplates = new ToolSTGroupFile(langDir+"/"+language+".stg"); 187 baseTemplates = coreTemplates; 188 189 // dynamically add subgroups that act like filters to apply to 190 // their supergroup. E.g., Java:Dbg:AST:ASTParser::ASTDbg. 191 String outputOption = (String)grammar.getOption("output"); 192 if ( outputOption!=null && outputOption.equals("AST") ) { 193 if ( debug && grammar.type!=Grammar.LEXER ) { 194 STGroup dbgTemplates = new ToolSTGroupFile(langDir+"/Dbg.stg"); 195 dbgTemplates.importTemplates(coreTemplates); 196 baseTemplates = dbgTemplates; 197 STGroup astTemplates = new ToolSTGroupFile(langDir+"/AST.stg"); 198 astTemplates.importTemplates(dbgTemplates); 199 STGroup astParserTemplates; 200 if ( grammar.type==Grammar.TREE_PARSER ) { 201 astParserTemplates = new ToolSTGroupFile(langDir+"/ASTTreeParser.stg"); 202 astParserTemplates.importTemplates(astTemplates); 203 } 204 else { 205 astParserTemplates = new ToolSTGroupFile(langDir+"/ASTParser.stg"); 206 astParserTemplates.importTemplates(astTemplates); 207 } 208 STGroup astDbgTemplates = new ToolSTGroupFile(langDir+"/ASTDbg.stg"); 209 astDbgTemplates.importTemplates(astParserTemplates); 210 templates = astDbgTemplates; 211 dbgTemplates.iterateAcrossValues = true; // ST v3 compatibility with Maps 212 astDbgTemplates.iterateAcrossValues = true; 213 astParserTemplates.iterateAcrossValues = true; 214 } 215 else { 216 STGroup astTemplates = new ToolSTGroupFile(langDir+"/AST.stg"); 217 astTemplates.importTemplates(coreTemplates); 218 STGroup astParserTemplates; 219 if ( grammar.type==Grammar.TREE_PARSER ) { 220 astParserTemplates = new ToolSTGroupFile(langDir+"/ASTTreeParser.stg"); 221 astParserTemplates.importTemplates(astTemplates); 222 } 223 else { 224 astParserTemplates = new ToolSTGroupFile(langDir+"/ASTParser.stg"); 225 astParserTemplates.importTemplates(astTemplates); 226 } 227 templates = astParserTemplates; 228 astTemplates.iterateAcrossValues = true; // ST v3 compatibility with Maps 229 astParserTemplates.iterateAcrossValues = true; 230 } 231 } 232 else if ( outputOption!=null && outputOption.equals("template") ) { 233 if ( debug && grammar.type!=Grammar.LEXER ) { 234 STGroup dbgTemplates = new ToolSTGroupFile(langDir+"/Dbg.stg"); 235 dbgTemplates.importTemplates(coreTemplates); 236 baseTemplates = dbgTemplates; 237 STGroup stTemplates = new ToolSTGroupFile(langDir+"/ST.stg"); 238 stTemplates.importTemplates(dbgTemplates); 239 templates = stTemplates; 240 dbgTemplates.iterateAcrossValues = true; 241 } 242 else { 243 STGroup stTemplates = new ToolSTGroupFile(langDir+"/ST.stg"); 244 stTemplates.importTemplates(coreTemplates); 245 templates = stTemplates; 246 } 247 templates.iterateAcrossValues = true; // ST v3 compatibility with Maps 248 } 249 else if ( debug && grammar.type!=Grammar.LEXER ) { 250 STGroup dbgTemplates = new ToolSTGroupFile(langDir+"/Dbg.stg"); 251 dbgTemplates.importTemplates(coreTemplates); 252 templates = dbgTemplates; 253 baseTemplates = templates; 254 baseTemplates.iterateAcrossValues = true; // ST v3 compatibility with Maps 255 } 256 else { 257 templates = coreTemplates; 258 coreTemplates.iterateAcrossValues = true; // ST v3 compatibility with Maps 259 } 260 } 261 262 /** Given the grammar to which we are attached, walk the AST associated 263 * with that grammar to create NFAs. Then create the DFAs for all 264 * decision points in the grammar by converting the NFAs to DFAs. 265 * Finally, walk the AST again to generate code. 266 * 267 * Either 1 or 2 files are written: 268 * 269 * recognizer: the main parser/lexer/treewalker item 270 * header file: language like C/C++ need extern definitions 271 * 272 * The target, such as JavaTarget, dictates which files get written. 273 */ 274 public ST genRecognizer() { 275 //System.out.println("### generate "+grammar.name+" recognizer"); 276 // LOAD OUTPUT TEMPLATES 277 loadTemplates(language); 278 if ( templates==null ) { 279 return null; 280 } 281 282 // CREATE NFA FROM GRAMMAR, CREATE DFA FROM NFA 283 if ( ErrorManager.doNotAttemptAnalysis() ) { 284 return null; 285 } 286 target.performGrammarAnalysis(this, grammar); 287 288 289 // some grammar analysis errors will not yield reliable DFA 290 if ( ErrorManager.doNotAttemptCodeGen() ) { 291 return null; 292 } 293 294 // OPTIMIZE DFA 295 DFAOptimizer optimizer = new DFAOptimizer(grammar); 296 optimizer.optimize(); 297 298 // OUTPUT FILE (contains recognizerST) 299 outputFileST = templates.getInstanceOf("outputFile"); 300 301 // HEADER FILE 302 if ( templates.isDefined("headerFile") ) { 303 headerFileST = templates.getInstanceOf("headerFile"); 304 } 305 else { 306 // create a dummy to avoid null-checks all over code generator 307 headerFileST = new ST(templates,"xyz"); 308 headerFileST.add("cyclicDFAs", (Object)null); // it normally sees this from outputFile 309 //headerFileST.impl.name = "dummy-header-file"; 310 } 311 312 boolean filterMode = grammar.getOption("filter")!=null && 313 grammar.getOption("filter").equals("true"); 314 boolean canBacktrack = grammar.getSyntacticPredicates()!=null || 315 grammar.composite.getRootGrammar().atLeastOneBacktrackOption || 316 filterMode; 317 318 // TODO: move this down further because generating the recognizer 319 // alters the model with info on who uses predefined properties etc... 320 // The actions here might refer to something. 321 322 // The only two possible output files are available at this point. 323 // Verify action scopes are ok for target and dump actions into output 324 // Templates can say <actions.parser.header> for example. 325 Map<String, Map<String, Object>> actions = grammar.getActions(); 326 verifyActionScopesOkForTarget(actions); 327 // translate $x::y references 328 translateActionAttributeReferences(actions); 329 330 ST gateST = templates.getInstanceOf("actionGate"); 331 if ( filterMode ) { 332 // if filtering, we need to set actions to execute at backtracking 333 // level 1 not 0. 334 gateST = templates.getInstanceOf("filteringActionGate"); 335 } 336 grammar.setSynPredGateIfNotAlready(gateST); 337 338 headerFileST.add("actions", actions); 339 outputFileST.add("actions", actions); 340 341 headerFileST.add("buildTemplate", grammar.buildTemplate()); 342 outputFileST.add("buildTemplate", grammar.buildTemplate()); 343 headerFileST.add("buildAST", grammar.buildAST()); 344 outputFileST.add("buildAST", grammar.buildAST()); 345 346 outputFileST.add("rewriteMode", grammar.rewriteMode()); 347 headerFileST.add("rewriteMode", grammar.rewriteMode()); 348 349 outputFileST.add("backtracking", canBacktrack); 350 headerFileST.add("backtracking", canBacktrack); 351 // turn on memoize attribute at grammar level so we can create ruleMemo. 352 // each rule has memoize attr that hides this one, indicating whether 353 // it needs to save results 354 String memoize = (String)grammar.getOption("memoize"); 355 outputFileST.add("memoize", 356 (grammar.atLeastOneRuleMemoizes || 357 memoize != null && memoize.equals("true") && 358 canBacktrack)); 359 headerFileST.add("memoize", 360 (grammar.atLeastOneRuleMemoizes || 361 memoize != null && memoize.equals("true") && 362 canBacktrack)); 363 364 365 outputFileST.add("trace", trace); 366 headerFileST.add("trace", trace); 367 368 outputFileST.add("profile", profile); 369 headerFileST.add("profile", profile); 370 371 // RECOGNIZER 372 if ( grammar.type==Grammar.LEXER ) { 373 recognizerST = templates.getInstanceOf("lexer"); 374 outputFileST.add("LEXER", true); 375 headerFileST.add("LEXER", true); 376 recognizerST.add("filterMode", 377 filterMode); 378 } 379 else if ( grammar.type==Grammar.PARSER || 380 grammar.type==Grammar.COMBINED ) 381 { 382 recognizerST = templates.getInstanceOf("parser"); 383 outputFileST.add("PARSER", true); 384 headerFileST.add("PARSER", true); 385 } 386 else { 387 recognizerST = templates.getInstanceOf("treeParser"); 388 outputFileST.add("TREE_PARSER", true); 389 headerFileST.add("TREE_PARSER", true); 390 recognizerST.add("filterMode", 391 filterMode); 392 } 393 outputFileST.add("recognizer", recognizerST); 394 headerFileST.add("recognizer", recognizerST); 395 outputFileST.add("actionScope", 396 grammar.getDefaultActionScope(grammar.type)); 397 headerFileST.add("actionScope", 398 grammar.getDefaultActionScope(grammar.type)); 399 400 String targetAppropriateFileNameString = 401 target.getTargetStringLiteralFromString(grammar.getFileName()); 402 outputFileST.add("fileName", targetAppropriateFileNameString); 403 headerFileST.add("fileName", targetAppropriateFileNameString); 404 outputFileST.add("ANTLRVersion", tool.VERSION); 405 headerFileST.add("ANTLRVersion", tool.VERSION); 406 outputFileST.add("generatedTimestamp", Tool.getCurrentTimeStamp()); 407 headerFileST.add("generatedTimestamp", Tool.getCurrentTimeStamp()); 408 409 // GENERATE RECOGNIZER 410 // Walk the AST holding the input grammar, this time generating code 411 // Decisions are generated by using the precomputed DFAs 412 // Fill in the various templates with data 413 CodeGenTreeWalker gen = new CodeGenTreeWalker(new CommonTreeNodeStream(grammar.getGrammarTree())); 414 try { 415 gen.grammar_( 416 grammar, 417 recognizerST, 418 outputFileST, 419 headerFileST); 420 } 421 catch (RecognitionException re) { 422 ErrorManager.error(ErrorManager.MSG_BAD_AST_STRUCTURE, 423 re); 424 } 425 426 genTokenTypeConstants(recognizerST); 427 genTokenTypeConstants(outputFileST); 428 genTokenTypeConstants(headerFileST); 429 430 if ( grammar.type!=Grammar.LEXER ) { 431 genTokenTypeNames(recognizerST); 432 genTokenTypeNames(outputFileST); 433 genTokenTypeNames(headerFileST); 434 } 435 436 // Now that we know what synpreds are used, we can set into template 437 Set<String> synpredNames = null; 438 if ( grammar.synPredNamesUsedInDFA.size()>0 ) { 439 synpredNames = grammar.synPredNamesUsedInDFA; 440 } 441 outputFileST.add("synpreds", synpredNames); 442 headerFileST.add("synpreds", synpredNames); 443 444 // all recognizers can see Grammar object 445 recognizerST.add("grammar", grammar); 446 447 // do not render templates to disk if errors occurred 448 if ( ErrorManager.getErrorState().errors > 0 ) { 449 return null; 450 } 451 452 if (LAUNCH_ST_INSPECTOR) { 453 outputFileST.inspect(); 454 if ( templates.isDefined("headerFile") ) headerFileST.inspect(); 455 } 456 457 // WRITE FILES 458 try { 459 target.genRecognizerFile(tool,this,grammar,outputFileST); 460 if ( templates.isDefined("headerFile") ) { 461 ST extST = templates.getInstanceOf("headerFileExtension"); 462 target.genRecognizerHeaderFile(tool,this,grammar,headerFileST,extST.render()); 463 } 464 // write out the vocab interchange file; used by antlr, 465 // does not change per target 466 ST tokenVocabSerialization = genTokenVocabOutput(); 467 String vocabFileName = getVocabFileName(); 468 if ( vocabFileName!=null ) { 469 write(tokenVocabSerialization, vocabFileName); 470 } 471 //System.out.println(outputFileST.getDOTForDependencyGraph(false)); 472 } 473 catch (IOException ioe) { 474 ErrorManager.error(ErrorManager.MSG_CANNOT_WRITE_FILE, ioe); 475 } 476 /* 477 System.out.println("num obj.prop refs: "+ ASTExpr.totalObjPropRefs); 478 System.out.println("num reflection lookups: "+ ASTExpr.totalReflectionLookups); 479 */ 480 481 return outputFileST; 482 } 483 484 /** Some targets will have some extra scopes like C++ may have 485 * '@headerfile:name {action}' or something. Make sure the 486 * target likes the scopes in action table. 487 */ 488 protected void verifyActionScopesOkForTarget(Map<String, Map<String, Object>> actions) { 489 for (Map.Entry<String, Map<String, Object>> entry : actions.entrySet()) { 490 String scope = entry.getKey(); 491 if ( !target.isValidActionScope(grammar.type, scope) ) { 492 // get any action from the scope to get error location 493 Map<String, Object> scopeActions = entry.getValue(); 494 GrammarAST actionAST = 495 (GrammarAST)scopeActions.values().iterator().next(); 496 ErrorManager.grammarError( 497 ErrorManager.MSG_INVALID_ACTION_SCOPE,grammar, 498 actionAST.getToken(),scope, 499 grammar.getGrammarTypeString()); 500 } 501 } 502 } 503 504 /** Actions may reference $x::y attributes, call translateAction on 505 * each action and replace that action in the Map. 506 */ 507 protected void translateActionAttributeReferences(Map<String, Map<String, Object>> actions) { 508 for (Map.Entry<String, Map<String, Object>> entry : actions.entrySet()) { 509 Map<String, Object> scopeActions = entry.getValue(); 510 translateActionAttributeReferencesForSingleScope(null,scopeActions); 511 } 512 } 513 514 /** Use for translating rule @init{...} actions that have no scope */ 515 public void translateActionAttributeReferencesForSingleScope( 516 Rule r, 517 Map<String, Object> scopeActions) 518 { 519 String ruleName=null; 520 if ( r!=null ) { 521 ruleName = r.name; 522 } 523 for (Map.Entry<String, Object> entry : scopeActions.entrySet()) { 524 String name = entry.getKey(); 525 GrammarAST actionAST = (GrammarAST)entry.getValue(); 526 List<?> chunks = translateAction(ruleName,actionAST); 527 scopeActions.put(name, chunks); // replace with translation 528 } 529 } 530 531 /** Error recovery in ANTLR recognizers. 532 * 533 * Based upon original ideas: 534 * 535 * Algorithms + Data Structures = Programs by Niklaus Wirth 536 * 537 * and 538 * 539 * A note on error recovery in recursive descent parsers: 540 * http://portal.acm.org/citation.cfm?id=947902.947905 541 * 542 * Later, Josef Grosch had some good ideas: 543 * Efficient and Comfortable Error Recovery in Recursive Descent Parsers: 544 * ftp://www.cocolab.com/products/cocktail/doca4.ps/ell.ps.zip 545 * 546 * Like Grosch I implemented local FOLLOW sets that are combined at run-time 547 * upon error to avoid parsing overhead. 548 */ 549 public void generateLocalFOLLOW(GrammarAST referencedElementNode, 550 String referencedElementName, 551 String enclosingRuleName, 552 int elementIndex) 553 { 554 /* 555 System.out.println("compute FOLLOW "+grammar.name+"."+referencedElementNode.toString()+ 556 " for "+referencedElementName+"#"+elementIndex +" in "+ 557 enclosingRuleName+ 558 " line="+referencedElementNode.getLine()); 559 */ 560 NFAState followingNFAState = referencedElementNode.followingNFAState; 561 LookaheadSet follow = null; 562 if ( followingNFAState!=null ) { 563 // compute follow for this element and, as side-effect, track 564 // the rule LOOK sensitivity. 565 follow = grammar.FIRST(followingNFAState); 566 } 567 568 if ( follow==null ) { 569 ErrorManager.internalError("no follow state or cannot compute follow"); 570 follow = new LookaheadSet(); 571 } 572 if ( follow.member(Label.EOF) ) { 573 // TODO: can we just remove? Seems needed here: 574 // compilation_unit : global_statement* EOF 575 // Actually i guess we resync to EOF regardless 576 follow.remove(Label.EOF); 577 } 578 //System.out.println(" "+follow); 579 580 List<Integer> tokenTypeList; 581 long[] words; 582 if ( follow.tokenTypeSet==null ) { 583 words = new long[1]; 584 tokenTypeList = new ArrayList<Integer>(); 585 } 586 else { 587 BitSet bits = BitSet.of(follow.tokenTypeSet); 588 words = bits.toPackedArray(); 589 tokenTypeList = follow.tokenTypeSet.toList(); 590 } 591 // use the target to convert to hex strings (typically) 592 String[] wordStrings = new String[words.length]; 593 for (int j = 0; j < words.length; j++) { 594 long w = words[j]; 595 wordStrings[j] = target.getTarget64BitStringFromValue(w); 596 } 597 recognizerST.addAggr("bitsets.{name,inName,bits,tokenTypes,tokenIndex}", 598 referencedElementName, 599 enclosingRuleName, 600 wordStrings, 601 tokenTypeList, 602 Utils.integer(elementIndex)); 603 outputFileST.addAggr("bitsets.{name,inName,bits,tokenTypes,tokenIndex}", 604 referencedElementName, 605 enclosingRuleName, 606 wordStrings, 607 tokenTypeList, 608 Utils.integer(elementIndex)); 609 headerFileST.addAggr("bitsets.{name,inName,bits,tokenTypes,tokenIndex}", 610 referencedElementName, 611 enclosingRuleName, 612 wordStrings, 613 tokenTypeList, 614 Utils.integer(elementIndex)); 615 } 616 617 // L O O K A H E A D D E C I S I O N G E N E R A T I O N 618 619 /** Generate code that computes the predicted alt given a DFA. The 620 * recognizerST can be either the main generated recognizerTemplate 621 * for storage in the main parser file or a separate file. It's up to 622 * the code that ultimately invokes the codegen.g grammar rule. 623 * 624 * Regardless, the output file and header file get a copy of the DFAs. 625 */ 626 public ST genLookaheadDecision(ST recognizerST, 627 DFA dfa) 628 { 629 ST decisionST; 630 // If we are doing inline DFA and this one is acyclic and LL(*) 631 // I have to check for is-non-LL(*) because if non-LL(*) the cyclic 632 // check is not done by DFA.verify(); that is, verify() avoids 633 // doesStateReachAcceptState() if non-LL(*) 634 if ( dfa.canInlineDecision() ) { 635 decisionST = 636 acyclicDFAGenerator.genFixedLookaheadDecision(getTemplates(), dfa); 637 } 638 else { 639 // generate any kind of DFA here (cyclic or acyclic) 640 dfa.createStateTables(this); 641 outputFileST.add("cyclicDFAs", dfa); 642 headerFileST.add("cyclicDFAs", dfa); 643 decisionST = templates.getInstanceOf("dfaDecision"); 644 String description = dfa.getNFADecisionStartState().getDescription(); 645 description = target.getTargetStringLiteralFromString(description); 646 if ( description!=null ) { 647 decisionST.add("description", description); 648 } 649 decisionST.add("decisionNumber", 650 Utils.integer(dfa.getDecisionNumber())); 651 } 652 return decisionST; 653 } 654 655 /** A special state is huge (too big for state tables) or has a predicated 656 * edge. Generate a simple if-then-else. Cannot be an accept state as 657 * they have no emanating edges. Don't worry about switch vs if-then-else 658 * because if you get here, the state is super complicated and needs an 659 * if-then-else. This is used by the new DFA scheme created June 2006. 660 */ 661 public ST generateSpecialState(DFAState s) { 662 ST stateST; 663 stateST = templates.getInstanceOf("cyclicDFAState"); 664 stateST.add("needErrorClause", true); 665 stateST.add("semPredState", 666 s.isResolvedWithPredicates()); 667 stateST.add("stateNumber", s.stateNumber); 668 stateST.add("decisionNumber", s.dfa.decisionNumber); 669 670 boolean foundGatedPred = false; 671 ST eotST = null; 672 for (int i = 0; i < s.getNumberOfTransitions(); i++) { 673 Transition edge = s.transition(i); 674 ST edgeST; 675 if ( edge.label.getAtom()==Label.EOT ) { 676 // this is the default clause; has to held until last 677 edgeST = templates.getInstanceOf("eotDFAEdge"); 678 stateST.remove("needErrorClause"); 679 eotST = edgeST; 680 } 681 else { 682 edgeST = templates.getInstanceOf("cyclicDFAEdge"); 683 ST exprST = 684 genLabelExpr(templates,edge,1); 685 edgeST.add("labelExpr", exprST); 686 } 687 edgeST.add("edgeNumber", Utils.integer(i + 1)); 688 edgeST.add("targetStateNumber", 689 Utils.integer(edge.target.stateNumber)); 690 // stick in any gated predicates for any edge if not already a pred 691 if ( !edge.label.isSemanticPredicate() ) { 692 DFAState t = (DFAState)edge.target; 693 SemanticContext preds = t.getGatedPredicatesInNFAConfigurations(); 694 if ( preds!=null ) { 695 foundGatedPred = true; 696 ST predST = preds.genExpr(this, 697 getTemplates(), 698 t.dfa); 699 edgeST.add("predicates", predST.render()); 700 } 701 } 702 if ( edge.label.getAtom()!=Label.EOT ) { 703 stateST.add("edges", edgeST); 704 } 705 } 706 if ( foundGatedPred ) { 707 // state has >= 1 edge with a gated pred (syn or sem) 708 // must rewind input first, set flag. 709 stateST.add("semPredState", foundGatedPred); 710 } 711 if ( eotST!=null ) { 712 stateST.add("edges", eotST); 713 } 714 return stateST; 715 } 716 717 /** Generate an expression for traversing an edge. */ 718 protected ST genLabelExpr(STGroup templates, 719 Transition edge, 720 int k) 721 { 722 Label label = edge.label; 723 if ( label.isSemanticPredicate() ) { 724 return genSemanticPredicateExpr(templates, edge); 725 } 726 if ( label.isSet() ) { 727 return genSetExpr(templates, label.getSet(), k, true); 728 } 729 // must be simple label 730 ST eST = templates.getInstanceOf("lookaheadTest"); 731 eST.add("atom", getTokenTypeAsTargetLabel(label.getAtom())); 732 eST.add("atomAsInt", Utils.integer(label.getAtom())); 733 eST.add("k", Utils.integer(k)); 734 return eST; 735 } 736 737 protected ST genSemanticPredicateExpr(STGroup templates, 738 Transition edge) 739 { 740 DFA dfa = ((DFAState)edge.target).dfa; // which DFA are we in 741 Label label = edge.label; 742 SemanticContext semCtx = label.getSemanticContext(); 743 return semCtx.genExpr(this,templates,dfa); 744 } 745 746 /** For intervals such as [3..3, 30..35], generate an expression that 747 * tests the lookahead similar to LA(1)==3 || (LA(1)>=30&&LA(1)<=35) 748 */ 749 public ST genSetExpr(STGroup templates, 750 IntSet set, 751 int k, 752 boolean partOfDFA) 753 { 754 if ( !(set instanceof IntervalSet) ) { 755 throw new IllegalArgumentException("unable to generate expressions for non IntervalSet objects"); 756 } 757 IntervalSet iset = (IntervalSet)set; 758 if ( iset.getIntervals()==null || iset.getIntervals().isEmpty() ) { 759 ST emptyST = new ST(templates, ""); 760 emptyST.impl.name = "empty-set-expr"; 761 return emptyST; 762 } 763 String testSTName = "lookaheadTest"; 764 String testRangeSTName = "lookaheadRangeTest"; 765 String testSetSTName = "lookaheadSetTest"; 766 String varSTName = "lookaheadVarName"; 767 if ( !partOfDFA ) { 768 testSTName = "isolatedLookaheadTest"; 769 testRangeSTName = "isolatedLookaheadRangeTest"; 770 testSetSTName = "isolatedLookaheadSetTest"; 771 varSTName = "isolatedLookaheadVarName"; 772 } 773 ST setST = templates.getInstanceOf("setTest"); 774 // If the SetTest template exists, separate the ranges: 775 // flatten the small ones into one list and make that a range, 776 // and leave the others as they are. 777 if ( templates.isDefined(testSetSTName) ) { 778 // Flatten the IntervalSet into a list of integers. 779 ST sST = templates.getInstanceOf(testSetSTName); 780 Iterator<Interval> iter = iset.getIntervals().iterator(); 781 int rangeNumber = 1; 782 while (iter.hasNext()) { 783 Interval I = iter.next(); 784 int a = I.a; 785 int b = I.b; 786 // Not flattening the large ranges helps us avoid making a 787 // set that contains 90% of Unicode when we could just use 788 // a simple range like (LA(1)>=123 && LA(1)<=65535). 789 // This flattens all ranges of length 4 or less. 790 if (b - a < 4) { 791 for (int i = a; i <= b; i++) { 792 sST.add("values", getTokenTypeAsTargetLabel(i)); 793 sST.add("valuesAsInt", Utils.integer(i)); 794 } 795 } else { 796 ST eST = templates.getInstanceOf(testRangeSTName); 797 eST.add("lower", getTokenTypeAsTargetLabel(a)); 798 eST.add("lowerAsInt", Utils.integer(a)); 799 eST.add("upper", getTokenTypeAsTargetLabel(b)); 800 eST.add("upperAsInt", Utils.integer(b)); 801 eST.add("rangeNumber", Utils.integer(rangeNumber)); 802 eST.add("k", Utils.integer(k)); 803 setST.add("ranges", eST); 804 rangeNumber++; 805 } 806 } 807 sST.add("k", Utils.integer(k)); 808 setST.add("ranges", sST); 809 return setST; 810 } 811 Iterator<Interval> iter = iset.getIntervals().iterator(); 812 int rangeNumber = 1; 813 while (iter.hasNext()) { 814 Interval I = iter.next(); 815 int a = I.a; 816 int b = I.b; 817 ST eST; 818 if ( a==b ) { 819 eST = templates.getInstanceOf(testSTName); 820 eST.add("atom", getTokenTypeAsTargetLabel(a)); 821 eST.add("atomAsInt", Utils.integer(a)); 822 //eST.add("k",Utils.integer(k)); 823 } 824 else { 825 eST = templates.getInstanceOf(testRangeSTName); 826 eST.add("lower", getTokenTypeAsTargetLabel(a)); 827 eST.add("lowerAsInt", Utils.integer(a)); 828 eST.add("upper", getTokenTypeAsTargetLabel(b)); 829 eST.add("upperAsInt", Utils.integer(b)); 830 eST.add("rangeNumber", Utils.integer(rangeNumber)); 831 } 832 eST.add("k", Utils.integer(k)); 833 setST.add("ranges", eST); 834 rangeNumber++; 835 } 836 return setST; 837 } 838 839 // T O K E N D E F I N I T I O N G E N E R A T I O N 840 841 /** Set attributes tokens and literals attributes in the incoming 842 * code template. This is not the token vocab interchange file, but 843 * rather a list of token type ID needed by the recognizer. 844 */ 845 protected void genTokenTypeConstants(ST code) { 846 // make constants for the token types 847 for (String tokenID : grammar.getTokenIDs()) { 848 int tokenType = grammar.getTokenType(tokenID); 849 if ( tokenType==Label.EOF || 850 tokenType>=Label.MIN_TOKEN_TYPE ) 851 { 852 // don't do FAUX labels 'cept EOF 853 code.addAggr("tokens.{name,type}", tokenID, Utils.integer(tokenType)); 854 } 855 } 856 } 857 858 /** Generate a token names table that maps token type to a printable 859 * name: either the label like INT or the literal like "begin". 860 */ 861 protected void genTokenTypeNames(ST code) { 862 for (int t=Label.MIN_TOKEN_TYPE; t<=grammar.getMaxTokenType(); t++) { 863 String tokenName = grammar.getTokenDisplayName(t); 864 if ( tokenName!=null ) { 865 tokenName=target.getTargetStringLiteralFromString(tokenName, true); 866 code.add("tokenNames", tokenName); 867 } 868 } 869 } 870 871 /** Get a meaningful name for a token type useful during code generation. 872 * Literals without associated names are converted to the string equivalent 873 * of their integer values. Used to generate x==ID and x==34 type comparisons 874 * etc... Essentially we are looking for the most obvious way to refer 875 * to a token type in the generated code. If in the lexer, return the 876 * char literal translated to the target language. For example, ttype=10 877 * will yield '\n' from the getTokenDisplayName method. That must 878 * be converted to the target languages literals. For most C-derived 879 * languages no translation is needed. 880 */ 881 public String getTokenTypeAsTargetLabel(int ttype) { 882 if ( grammar.type==Grammar.LEXER ) { 883 String name = grammar.getTokenDisplayName(ttype); 884 return target.getTargetCharLiteralFromANTLRCharLiteral(this,name); 885 } 886 return target.getTokenTypeAsTargetLabel(this,ttype); 887 } 888 889 /** Generate a token vocab file with all the token names/types. For example: 890 * ID=7 891 * FOR=8 892 * 'for'=8 893 * 894 * This is independent of the target language; used by antlr internally 895 */ 896 protected ST genTokenVocabOutput() { 897 ST vocabFileST = new ST(vocabFilePattern); 898 vocabFileST.add("literals",(Object)null); // "define" literals arg 899 vocabFileST.add("tokens",(Object)null); 900 vocabFileST.impl.name = "vocab-file"; 901 // make constants for the token names 902 for (String tokenID : grammar.getTokenIDs()) { 903 int tokenType = grammar.getTokenType(tokenID); 904 if ( tokenType>=Label.MIN_TOKEN_TYPE ) { 905 vocabFileST.addAggr("tokens.{name,type}", tokenID, Utils.integer(tokenType)); 906 } 907 } 908 909 // now dump the strings 910 for (String literal : grammar.getStringLiterals()) { 911 int tokenType = grammar.getTokenType(literal); 912 if ( tokenType>=Label.MIN_TOKEN_TYPE ) { 913 vocabFileST.addAggr("tokens.{name,type}", literal, Utils.integer(tokenType)); 914 } 915 } 916 917 return vocabFileST; 918 } 919 920 public List<? extends Object> translateAction(String ruleName, 921 GrammarAST actionTree) 922 { 923 if ( actionTree.getType()==ANTLRParser.ARG_ACTION ) { 924 return translateArgAction(ruleName, actionTree); 925 } 926 ActionTranslator translator = new ActionTranslator(this,ruleName,actionTree); 927 List<Object> chunks = translator.translateToChunks(); 928 chunks = target.postProcessAction(chunks, actionTree.token); 929 return chunks; 930 } 931 932 /** Translate an action like [3,"foo",a[3]] and return a List of the 933 * translated actions. Because actions are themselves translated to a list 934 * of chunks, must cat together into a ST>. Don't translate 935 * to strings early as we need to eval templates in context. 936 */ 937 public List<ST> translateArgAction(String ruleName, 938 GrammarAST actionTree) 939 { 940 String actionText = actionTree.token.getText(); 941 List<String> args = getListOfArgumentsFromAction(actionText,','); 942 List<ST> translatedArgs = new ArrayList<ST>(); 943 for (String arg : args) { 944 if ( arg!=null ) { 945 Token actionToken = 946 new CommonToken(ANTLRParser.ACTION,arg); 947 ActionTranslator translator = 948 new ActionTranslator(this,ruleName, 949 actionToken, 950 actionTree.outerAltNum); 951 List<Object> chunks = translator.translateToChunks(); 952 chunks = target.postProcessAction(chunks, actionToken); 953 ST catST = new ST(templates, "<chunks>"); 954 catST.add("chunks", chunks); 955 translatedArgs.add(catST); 956 } 957 } 958 if ( translatedArgs.isEmpty() ) { 959 return null; 960 } 961 return translatedArgs; 962 } 963 964 public static List<String> getListOfArgumentsFromAction(String actionText, 965 int separatorChar) 966 { 967 List<String> args = new ArrayList<String>(); 968 getListOfArgumentsFromAction(actionText, 0, -1, separatorChar, args); 969 return args; 970 } 971 972 /** Given an arg action like 973 * 974 * [x, (*a).foo(21,33), 3.2+1, '\n', 975 * "a,oo\nick", {bl, "fdkj"eck}, ["cat\n,", x, 43]] 976 * 977 * convert to a list of arguments. Allow nested square brackets etc... 978 * Set separatorChar to ';' or ',' or whatever you want. 979 */ 980 public static int getListOfArgumentsFromAction(String actionText, 981 int start, 982 int targetChar, 983 int separatorChar, 984 List<String> args) 985 { 986 if ( actionText==null ) { 987 return -1; 988 } 989 actionText = actionText.replaceAll("//.*\n", ""); 990 int n = actionText.length(); 991 //System.out.println("actionText@"+start+"->"+(char)targetChar+"="+actionText.substring(start,n)); 992 int p = start; 993 int last = p; 994 while ( p<n && actionText.charAt(p)!=targetChar ) { 995 int c = actionText.charAt(p); 996 switch ( c ) { 997 case '\'' : 998 p++; 999 while ( p<n && actionText.charAt(p)!='\'' ) { 1000 if ( actionText.charAt(p)=='\\' && (p+1)<n && 1001 actionText.charAt(p+1)=='\'' ) 1002 { 1003 p++; // skip escaped quote 1004 } 1005 p++; 1006 } 1007 p++; 1008 break; 1009 case '"' : 1010 p++; 1011 while ( p<n && actionText.charAt(p)!='\"' ) { 1012 if ( actionText.charAt(p)=='\\' && (p+1)<n && 1013 actionText.charAt(p+1)=='\"' ) 1014 { 1015 p++; // skip escaped quote 1016 } 1017 p++; 1018 } 1019 p++; 1020 break; 1021 case '(' : 1022 p = getListOfArgumentsFromAction(actionText,p+1,')',separatorChar,args); 1023 break; 1024 case '{' : 1025 p = getListOfArgumentsFromAction(actionText,p+1,'}',separatorChar,args); 1026 break; 1027 case '<' : 1028 if ( actionText.indexOf('>',p+1)>=p ) { 1029 // do we see a matching '>' ahead? if so, hope it's a generic 1030 // and not less followed by expr with greater than 1031 p = getListOfArgumentsFromAction(actionText,p+1,'>',separatorChar,args); 1032 } 1033 else { 1034 p++; // treat as normal char 1035 } 1036 break; 1037 case '[' : 1038 p = getListOfArgumentsFromAction(actionText,p+1,']',separatorChar,args); 1039 break; 1040 default : 1041 if ( c==separatorChar && targetChar==-1 ) { 1042 String arg = actionText.substring(last, p); 1043 //System.out.println("arg="+arg); 1044 args.add(arg.trim()); 1045 last = p+1; 1046 } 1047 p++; 1048 break; 1049 } 1050 } 1051 if ( targetChar==-1 && p<=n ) { 1052 String arg = actionText.substring(last, p).trim(); 1053 //System.out.println("arg="+arg); 1054 if ( arg.length()>0 ) { 1055 args.add(arg.trim()); 1056 } 1057 } 1058 p++; 1059 return p; 1060 } 1061 1062 /** Given a template constructor action like %foo(a={...}) in 1063 * an action, translate it to the appropriate template constructor 1064 * from the templateLib. This translates a *piece* of the action. 1065 */ 1066 public ST translateTemplateConstructor(String ruleName, 1067 int outerAltNum, 1068 Token actionToken, 1069 String templateActionText) 1070 { 1071 // first, parse with antlr.g 1072 //System.out.println("translate template: "+templateActionText); 1073 ANTLRLexer lexer = new ANTLRLexer(new ANTLRStringStream(templateActionText)); 1074 lexer.setFileName(grammar.getFileName()); 1075 ANTLRParser parser = ANTLRParser.createParser(new CommonTokenStream(lexer)); 1076 parser.setFileName(grammar.getFileName()); 1077 ANTLRParser.rewrite_template_return parseResult = null; 1078 try { 1079 parseResult = parser.rewrite_template(); 1080 } 1081 catch (RecognitionException re) { 1082 ErrorManager.grammarError(ErrorManager.MSG_INVALID_TEMPLATE_ACTION, 1083 grammar, 1084 actionToken, 1085 templateActionText); 1086 } 1087 catch (Exception tse) { 1088 ErrorManager.internalError("can't parse template action",tse); 1089 } 1090 GrammarAST rewriteTree = parseResult.getTree(); 1091 1092 // then translate via codegen.g 1093 CodeGenTreeWalker gen = new CodeGenTreeWalker(new CommonTreeNodeStream(rewriteTree)); 1094 gen.init(grammar); 1095 gen.setCurrentRuleName(ruleName); 1096 gen.setOuterAltNum(outerAltNum); 1097 ST st = null; 1098 try { 1099 st = gen.rewrite_template(); 1100 } 1101 catch (RecognitionException re) { 1102 ErrorManager.error(ErrorManager.MSG_BAD_AST_STRUCTURE, 1103 re); 1104 } 1105 return st; 1106 } 1107 1108 1109 public void issueInvalidScopeError(String x, 1110 String y, 1111 Rule enclosingRule, 1112 Token actionToken, 1113 int outerAltNum) 1114 { 1115 //System.out.println("error $"+x+"::"+y); 1116 Rule r = grammar.getRule(x); 1117 AttributeScope scope = grammar.getGlobalScope(x); 1118 if ( scope==null ) { 1119 if ( r!=null ) { 1120 scope = r.ruleScope; // if not global, might be rule scope 1121 } 1122 } 1123 if ( scope==null ) { 1124 ErrorManager.grammarError(ErrorManager.MSG_UNKNOWN_DYNAMIC_SCOPE, 1125 grammar, 1126 actionToken, 1127 x); 1128 } 1129 else if ( scope.getAttribute(y)==null ) { 1130 ErrorManager.grammarError(ErrorManager.MSG_UNKNOWN_DYNAMIC_SCOPE_ATTRIBUTE, 1131 grammar, 1132 actionToken, 1133 x, 1134 y); 1135 } 1136 } 1137 1138 public void issueInvalidAttributeError(String x, 1139 String y, 1140 Rule enclosingRule, 1141 Token actionToken, 1142 int outerAltNum) 1143 { 1144 //System.out.println("error $"+x+"."+y); 1145 if ( enclosingRule==null ) { 1146 // action not in a rule 1147 ErrorManager.grammarError(ErrorManager.MSG_ATTRIBUTE_REF_NOT_IN_RULE, 1148 grammar, 1149 actionToken, 1150 x, 1151 y); 1152 return; 1153 } 1154 1155 // action is in a rule 1156 Grammar.LabelElementPair label = enclosingRule.getRuleLabel(x); 1157 1158 if ( label!=null || enclosingRule.getRuleRefsInAlt(x, outerAltNum)!=null ) { 1159 // $rulelabel.attr or $ruleref.attr; must be unknown attr 1160 String refdRuleName = x; 1161 if ( label!=null ) { 1162 refdRuleName = enclosingRule.getRuleLabel(x).referencedRuleName; 1163 } 1164 Rule refdRule = grammar.getRule(refdRuleName); 1165 AttributeScope scope = refdRule.getAttributeScope(y); 1166 if ( scope==null ) { 1167 ErrorManager.grammarError(ErrorManager.MSG_UNKNOWN_RULE_ATTRIBUTE, 1168 grammar, 1169 actionToken, 1170 refdRuleName, 1171 y); 1172 } 1173 else if ( scope.isParameterScope ) { 1174 ErrorManager.grammarError(ErrorManager.MSG_INVALID_RULE_PARAMETER_REF, 1175 grammar, 1176 actionToken, 1177 refdRuleName, 1178 y); 1179 } 1180 else if ( scope.isDynamicRuleScope ) { 1181 ErrorManager.grammarError(ErrorManager.MSG_INVALID_RULE_SCOPE_ATTRIBUTE_REF, 1182 grammar, 1183 actionToken, 1184 refdRuleName, 1185 y); 1186 } 1187 } 1188 1189 } 1190 1191 public void issueInvalidAttributeError(String x, 1192 Rule enclosingRule, 1193 Token actionToken, 1194 int outerAltNum) 1195 { 1196 //System.out.println("error $"+x); 1197 if ( enclosingRule==null ) { 1198 // action not in a rule 1199 ErrorManager.grammarError(ErrorManager.MSG_ATTRIBUTE_REF_NOT_IN_RULE, 1200 grammar, 1201 actionToken, 1202 x); 1203 return; 1204 } 1205 1206 // action is in a rule 1207 Grammar.LabelElementPair label = enclosingRule.getRuleLabel(x); 1208 AttributeScope scope = enclosingRule.getAttributeScope(x); 1209 1210 if ( label!=null || 1211 enclosingRule.getRuleRefsInAlt(x, outerAltNum)!=null || 1212 enclosingRule.name.equals(x) ) 1213 { 1214 ErrorManager.grammarError(ErrorManager.MSG_ISOLATED_RULE_SCOPE, 1215 grammar, 1216 actionToken, 1217 x); 1218 } 1219 else if ( scope!=null && scope.isDynamicRuleScope ) { 1220 ErrorManager.grammarError(ErrorManager.MSG_ISOLATED_RULE_ATTRIBUTE, 1221 grammar, 1222 actionToken, 1223 x); 1224 } 1225 else { 1226 ErrorManager.grammarError(ErrorManager.MSG_UNKNOWN_SIMPLE_ATTRIBUTE, 1227 grammar, 1228 actionToken, 1229 x); 1230 } 1231 } 1232 1233 // M I S C 1234 1235 public STGroup getTemplates() { 1236 return templates; 1237 } 1238 1239 public STGroup getBaseTemplates() { 1240 return baseTemplates; 1241 } 1242 1243 public void setDebug(boolean debug) { 1244 this.debug = debug; 1245 } 1246 1247 public void setTrace(boolean trace) { 1248 this.trace = trace; 1249 } 1250 1251 public void setProfile(boolean profile) { 1252 this.profile = profile; 1253 if ( profile ) { 1254 setDebug(true); // requires debug events 1255 } 1256 } 1257 1258 public ST getRecognizerST() { 1259 return outputFileST; 1260 } 1261 1262 /** Generate TParser.java and TLexer.java from T.g if combined, else 1263 * just use T.java as output regardless of type. 1264 */ 1265 public String getRecognizerFileName(String name, int type) { 1266 ST extST = templates.getInstanceOf("codeFileExtension"); 1267 String recognizerName = grammar.getRecognizerName(); 1268 return recognizerName+extST.render(); 1269 /* 1270 String suffix = ""; 1271 if ( type==Grammar.COMBINED || 1272 (type==Grammar.LEXER && !grammar.implicitLexer) ) 1273 { 1274 suffix = Grammar.grammarTypeToFileNameSuffix[type]; 1275 } 1276 return name+suffix+extST.toString(); 1277 */ 1278 } 1279 1280 /** What is the name of the vocab file generated for this grammar? 1281 * Returns null if no .tokens file should be generated. 1282 */ 1283 public String getVocabFileName() { 1284 if ( grammar.isBuiltFromString() ) { 1285 return null; 1286 } 1287 return grammar.name+VOCAB_FILE_EXTENSION; 1288 } 1289 1290 public void write(ST code, String fileName) throws IOException { 1291 //long start = System.currentTimeMillis(); 1292 Writer w = tool.getOutputFile(grammar, fileName); 1293 // Write the output to a StringWriter 1294 STWriter wr = new AutoIndentWriter(w); 1295 wr.setLineWidth(lineWidth); 1296 code.write(wr); 1297 w.close(); 1298 //long stop = System.currentTimeMillis(); 1299 //System.out.println("render time for "+fileName+": "+(int)(stop-start)+"ms"); 1300 } 1301 1302 /** You can generate a switch rather than if-then-else for a DFA state 1303 * if there are no semantic predicates and the number of edge label 1304 * values is small enough; e.g., don't generate a switch for a state 1305 * containing an edge label such as 20..52330 (the resulting byte codes 1306 * would overflow the method 65k limit probably). 1307 */ 1308 protected boolean canGenerateSwitch(DFAState s) { 1309 if ( !GENERATE_SWITCHES_WHEN_POSSIBLE ) { 1310 return false; 1311 } 1312 int size = 0; 1313 for (int i = 0; i < s.getNumberOfTransitions(); i++) { 1314 Transition edge = s.transition(i); 1315 if ( edge.label.isSemanticPredicate() ) { 1316 return false; 1317 } 1318 // can't do a switch if the edges are going to require predicates 1319 if ( edge.label.getAtom()==Label.EOT ) { 1320 int EOTPredicts = ((DFAState)edge.target).getUniquelyPredictedAlt(); 1321 if ( EOTPredicts==NFA.INVALID_ALT_NUMBER ) { 1322 // EOT target has to be a predicate then; no unique alt 1323 return false; 1324 } 1325 } 1326 // if target is a state with gated preds, we need to use preds on 1327 // this edge then to reach it. 1328 if ( ((DFAState)edge.target).getGatedPredicatesInNFAConfigurations()!=null ) { 1329 return false; 1330 } 1331 size += edge.label.getSet().size(); 1332 } 1333 if ( s.getNumberOfTransitions()<MIN_SWITCH_ALTS || 1334 size>MAX_SWITCH_CASE_LABELS ) { 1335 return false; 1336 } 1337 return true; 1338 } 1339 1340 /** Create a label to track a token / rule reference's result. 1341 * Technically, this is a place where I break model-view separation 1342 * as I am creating a variable name that could be invalid in a 1343 * target language, however, label ::= <ID><INT> is probably ok in 1344 * all languages we care about. 1345 */ 1346 public String createUniqueLabel(String name) { 1347 return new StringBuffer() 1348 .append(name).append(uniqueLabelNumber++).toString(); 1349 } 1350 } 1351
