1 ANTLR_BEGIN_NAMESPACE() 2 3 template<class ImplTraits> 4 Lexer<ImplTraits>::Lexer(ANTLR_UINT32 sizeHint, RecognizerSharedStateType* state) 5 :Lexer<ImplTraits>::RecognizerType(sizeHint, state) 6 ,m_input(NULL) 7 { 8 } 9 10 template<class ImplTraits> 11 Lexer<ImplTraits>::Lexer(ANTLR_UINT32 sizeHint, InputStreamType* input, RecognizerSharedStateType* state) 12 :Lexer<ImplTraits>::RecognizerType(sizeHint, state) 13 { 14 this->setCharStream(input); 15 } 16 17 template<class ImplTraits> 18 typename Lexer<ImplTraits>::InputStreamType* Lexer<ImplTraits>::get_input() const 19 { 20 return m_input; 21 } 22 23 template<class ImplTraits> 24 typename Lexer<ImplTraits>::IntStreamType* Lexer<ImplTraits>::get_istream() const 25 { 26 return m_input; 27 } 28 29 template<class ImplTraits> 30 typename Lexer<ImplTraits>::RecognizerType* Lexer<ImplTraits>::get_rec() 31 { 32 return this; 33 } 34 35 template<class ImplTraits> 36 typename Lexer<ImplTraits>::TokenSourceType* Lexer<ImplTraits>::get_tokSource() 37 { 38 return this; 39 } 40 41 template<class ImplTraits> 42 void Lexer<ImplTraits>::displayRecognitionError( ANTLR_UINT8** , ExceptionBaseType* ex) 43 { 44 StringStreamType err_stream; 45 46 // See if there is a 'filename' we can use 47 // 48 if( ex->getName().empty() ) 49 { 50 err_stream << "-unknown source-("; 51 } 52 else 53 { 54 err_stream << ex->get_streamName().c_str(); 55 err_stream << "("; 56 } 57 err_stream << ex->get_line() << ")"; 58 59 err_stream << ": lexer error " << ex->getName() << '(' << ex->getType() << ')' << " :\n\t" 60 << ex->get_message() << " at position [" << ex->get_line() << ", " 61 << ex->get_charPositionInLine()+1 << "], "; 62 63 { 64 ANTLR_UINT32 width; 65 66 width = ANTLR_UINT32_CAST(( (ANTLR_UINT8*)(m_input->get_data()) + 67 (m_input->size() )) - (ANTLR_UINT8*)( ex->get_index() )); 68 69 if (width >= 1) 70 { 71 if (isprint(ex->get_c() )) 72 { 73 err_stream << "near '" << (typename StringType::value_type) ex->get_c() << "' :\n"; 74 } 75 else 76 { 77 err_stream << "near char(" << std::hex << ex->get_c() << std::dec << ") :\n"; 78 } 79 err_stream << "\t"; 80 err_stream.width( width > 20 ? 20 : width ); 81 err_stream << (typename StringType::const_pointer)ex->get_index() << "\n"; 82 } 83 else 84 { 85 err_stream << "(end of input).\n\t This indicates a poorly specified lexer RULE\n\t or unterminated input element such as: \"STRING[\"]\n"; 86 err_stream << "\t The lexer was matching from line " 87 << this->get_state()->get_tokenStartLine() 88 << ", offset " << this->get_state()->get_tokenStartCharPositionInLine() 89 << ", which\n\t "; 90 width = ANTLR_UINT32_CAST(((ANTLR_UINT8*)(m_input->get_data() )+ 91 (m_input->size())) - 92 (ANTLR_UINT8*)(this->get_state()->get_tokenStartCharIndex() )); 93 94 if (width >= 1) 95 { 96 err_stream << "looks like this:\n\t\t"; 97 err_stream.width( width > 20 ? 20 : width ); 98 err_stream << (typename StringType::const_pointer)this->get_state()->get_tokenStartCharIndex() << "\n"; 99 } 100 else 101 { 102 err_stream << "is also the end of the line, so you must check your lexer rules\n"; 103 } 104 } 105 } 106 ImplTraits::displayRecognitionError( err_stream.str() ); 107 } 108 109 template<class ImplTraits> 110 void Lexer<ImplTraits>::fillExceptionData( ExceptionBaseType* ex ) 111 { 112 ex->set_c( m_input->_LA(1) ); /* Current input character */ 113 ex->set_line( m_input->get_line() ); /* Line number comes from stream */ 114 ex->set_charPositionInLine( m_input->get_charPositionInLine() ); /* Line offset also comes from the stream */ 115 ex->set_index( m_input->index() ); 116 ex->set_streamName( m_input->get_fileName() ); 117 ex->set_message( "Unexpected character" ); 118 } 119 120 template<class ImplTraits> 121 void Lexer<ImplTraits>::setCharStream(InputStreamType* input) 122 { 123 /* Install the input interface 124 */ 125 m_input = input; 126 127 /* Set the current token to nothing 128 */ 129 RecognizerSharedStateType* state = this->get_rec()->get_state(); 130 state->set_token_present( false ); 131 state->set_text(""); 132 state->set_tokenStartCharIndex(-1); 133 134 /* Copy the name of the char stream to the token source 135 */ 136 this->get_tokSource()->set_fileName( input->get_fileName() ); 137 } 138 139 template<class ImplTraits> 140 void Lexer<ImplTraits>::pushCharStream(InputStreamType* input) 141 { 142 // We have a stack, so we can save the current input stream 143 // into it. 144 // 145 this->get_istream()->mark(); 146 this->get_rec()->get_state()->get_streams().push(this->get_input()); 147 148 // And now we can install this new one 149 // 150 this->setCharStream(input); 151 } 152 153 template<class ImplTraits> 154 void Lexer<ImplTraits>::popCharStream() 155 { 156 InputStreamType* input; 157 158 // If we do not have a stream stack or we are already at the 159 // stack bottom, then do nothing. 160 // 161 typename RecognizerSharedStateType::StreamsType& streams = this->get_rec()->get_state()->get_streams(); 162 if ( streams.size() > 0) 163 { 164 // We just leave the current stream to its fate, we do not close 165 // it or anything as we do not know what the programmer intended 166 // for it. This method can always be overridden of course. 167 // So just find out what was currently saved on the stack and use 168 // that now, then pop it from the stack. 169 // 170 input = streams.top(); 171 streams.pop(); 172 173 // Now install the stream as the current one. 174 // 175 this->setCharStream(input); 176 this->get_istream()->rewindLast(); 177 } 178 return; 179 } 180 181 template<class ImplTraits> 182 void Lexer<ImplTraits>::emit(const CommonTokenType* token) 183 { 184 this->get_rec()->get_state()->set_token(token); 185 } 186 187 template<class ImplTraits> 188 typename Lexer<ImplTraits>::CommonTokenType* Lexer<ImplTraits>::emit() 189 { 190 /* We could check pointers to token factories and so on, but 191 * we are in code that we want to run as fast as possible 192 * so we are not checking any errors. So make sure you have installed an input stream before 193 * trying to emit a new token. 194 */ 195 RecognizerSharedStateType* state = this->get_rec()->get_state(); 196 state->set_token_present(true); 197 CommonTokenType* token = state->get_token(); 198 token->set_input( this->get_input() ); 199 200 /* Install the supplied information, and some other bits we already know 201 * get added automatically, such as the input stream it is associated with 202 * (though it can all be overridden of course) 203 */ 204 token->set_type( state->get_type() ); 205 token->set_channel( state->get_channel() ); 206 token->set_startIndex( state->get_tokenStartCharIndex() ); 207 token->set_stopIndex( this->getCharIndex() - 1 ); 208 token->set_line( state->get_tokenStartLine() ); 209 token->set_charPositionInLine( state->get_tokenStartCharPositionInLine() ); 210 211 token->set_tokText( state->get_text() ); 212 token->set_lineStart( this->get_input()->get_currentLine() ); 213 214 return token; 215 } 216 217 template<class ImplTraits> 218 Lexer<ImplTraits>::~Lexer() 219 { 220 // This may have ben a delegate or delegator lexer, in which case the 221 // state may already have been freed (and set to NULL therefore) 222 // so we ignore the state if we don't have it. 223 // 224 RecognizerSharedStateType* state = this->get_rec()->get_state(); 225 226 if ( state != NULL) 227 { 228 state->get_streams().clear(); 229 } 230 } 231 232 template<class ImplTraits> 233 bool Lexer<ImplTraits>::matchs(ANTLR_UCHAR* str ) 234 { 235 RecognizerSharedStateType* state = this->get_rec()->get_state(); 236 while (*str != ANTLR_STRING_TERMINATOR) 237 { 238 if ( this->get_istream()->_LA(1) != (*str)) 239 { 240 if ( state->get_backtracking() > 0) 241 { 242 state->set_failed(true); 243 return false; 244 } 245 246 this->exConstruct(); 247 state->set_failed( true ); 248 249 /* TODO: Implement exception creation more fully perhaps 250 */ 251 this->recover(); 252 return false; 253 } 254 255 /* Matched correctly, do consume it 256 */ 257 this->get_istream()->consume(); 258 str++; 259 260 } 261 /* Reset any failed indicator 262 */ 263 state->set_failed( false ); 264 return true; 265 } 266 267 template<class ImplTraits> 268 bool Lexer<ImplTraits>::matchc(ANTLR_UCHAR c) 269 { 270 if (this->get_istream()->_LA(1) == c) 271 { 272 /* Matched correctly, do consume it 273 */ 274 this->get_istream()->consume(); 275 276 /* Reset any failed indicator 277 */ 278 this->get_rec()->get_state()->set_failed( false ); 279 280 return true; 281 } 282 283 /* Failed to match, exception and recovery time. 284 */ 285 if(this->get_rec()->get_state()->get_backtracking() > 0) 286 { 287 this->get_rec()->get_state()->set_failed( true ); 288 return false; 289 } 290 291 this->exConstruct(); 292 293 /* TODO: Implement exception creation more fully perhaps 294 */ 295 this->recover(); 296 297 return false; 298 } 299 300 template<class ImplTraits> 301 bool Lexer<ImplTraits>::matchRange(ANTLR_UCHAR low, ANTLR_UCHAR high) 302 { 303 ANTLR_UCHAR c; 304 305 /* What is in the stream at the moment? 306 */ 307 c = this->get_istream()->_LA(1); 308 if ( c >= low && c <= high) 309 { 310 /* Matched correctly, consume it 311 */ 312 this->get_istream()->consume(); 313 314 /* Reset any failed indicator 315 */ 316 this->get_rec()->get_state()->set_failed( false ); 317 318 return true; 319 } 320 321 /* Failed to match, execption and recovery time. 322 */ 323 324 if (this->get_rec()->get_state()->get_backtracking() > 0) 325 { 326 this->get_rec()->get_state()->set_failed( true ); 327 return false; 328 } 329 330 this->exConstruct(); 331 332 /* TODO: Implement exception creation more fully 333 */ 334 this->recover(); 335 336 return false; 337 } 338 339 template<class ImplTraits> 340 void Lexer<ImplTraits>::matchAny() 341 { 342 this->get_istream()->consume(); 343 } 344 345 template<class ImplTraits> 346 void Lexer<ImplTraits>::recover() 347 { 348 this->get_istream()->consume(); 349 } 350 351 template<class ImplTraits> 352 ANTLR_UINT32 Lexer<ImplTraits>::getLine() 353 { 354 return this->get_input()->get_line(); 355 } 356 357 template<class ImplTraits> 358 ANTLR_MARKER Lexer<ImplTraits>::getCharIndex() 359 { 360 return this->get_istream()->index(); 361 } 362 363 template<class ImplTraits> 364 ANTLR_UINT32 Lexer<ImplTraits>::getCharPositionInLine() 365 { 366 return this->get_input()->get_charPositionInLine(); 367 } 368 369 template<class ImplTraits> 370 typename Lexer<ImplTraits>::StringType Lexer<ImplTraits>::getText() 371 { 372 RecognizerSharedStateType* state = this->get_rec()->get_state(); 373 if ( !state->get_text().empty() ) 374 { 375 return state->get_text(); 376 377 } 378 return this->get_input()->substr( state->get_tokenStartCharIndex(), 379 this->getCharIndex() - this->get_input()->get_charByteSize() 380 ); 381 } 382 383 template<class ImplTraits> 384 void Lexer<ImplTraits>::exConstruct() 385 { 386 new ANTLR_Exception<ImplTraits, RECOGNITION_EXCEPTION, InputStreamType>( this->get_rec(), "" ); 387 } 388 389 template< class ImplTraits> 390 typename Lexer<ImplTraits>::TokenType* Lexer<ImplTraits>::getMissingSymbol( IntStreamType*, 391 ExceptionBaseType*, 392 ANTLR_UINT32 , BitsetListType*) 393 { 394 return NULL; 395 } 396 397 template< class ImplTraits> 398 ANTLR_INLINE const typename Lexer<ImplTraits>::RecognizerType* Lexer<ImplTraits>::get_rec() const 399 { 400 return this; 401 } 402 403 template< class ImplTraits> 404 ANTLR_INLINE const typename Lexer<ImplTraits>::RecognizerType* Lexer<ImplTraits>::get_recognizer() const 405 { 406 return this->get_rec(); 407 } 408 409 template< class ImplTraits> 410 ANTLR_INLINE typename Lexer<ImplTraits>::RecognizerSharedStateType* Lexer<ImplTraits>::get_lexstate() const 411 { 412 return this->get_rec()->get_state(); 413 } 414 415 template< class ImplTraits> 416 ANTLR_INLINE void Lexer<ImplTraits>::set_lexstate( RecognizerSharedStateType* lexstate ) 417 { 418 this->get_rec()->set_state(lexstate); 419 } 420 421 template< class ImplTraits> 422 ANTLR_INLINE const typename Lexer<ImplTraits>::TokenSourceType* Lexer<ImplTraits>::get_tokSource() const 423 { 424 return this; 425 } 426 427 template< class ImplTraits> 428 ANTLR_INLINE typename Lexer<ImplTraits>::CommonTokenType* Lexer<ImplTraits>::get_ltoken() const 429 { 430 return this->get_lexstate()->token(); 431 } 432 433 template< class ImplTraits> 434 ANTLR_INLINE void Lexer<ImplTraits>::set_ltoken( const CommonTokenType* ltoken ) 435 { 436 this->get_lexstate()->set_token( ltoken ); 437 } 438 439 template< class ImplTraits> 440 ANTLR_INLINE bool Lexer<ImplTraits>::hasFailed() const 441 { 442 return this->get_lexstate()->get_failed(); 443 } 444 445 template< class ImplTraits> 446 ANTLR_INLINE ANTLR_INT32 Lexer<ImplTraits>::get_backtracking() const 447 { 448 return this->get_lexstate()->get_backtracking(); 449 } 450 451 template< class ImplTraits> 452 ANTLR_INLINE void Lexer<ImplTraits>::inc_backtracking() 453 { 454 this->get_lexstate()->inc_backtracking(); 455 } 456 457 template< class ImplTraits> 458 ANTLR_INLINE void Lexer<ImplTraits>::dec_backtracking() 459 { 460 this->get_lexstate()->dec_backtracking(); 461 } 462 463 template< class ImplTraits> 464 ANTLR_INLINE bool Lexer<ImplTraits>::get_failedflag() const 465 { 466 return this->get_lexstate()->get_failed(); 467 } 468 469 template< class ImplTraits> 470 ANTLR_INLINE void Lexer<ImplTraits>::set_failedflag( bool failed ) 471 { 472 this->get_lexstate()->set_failed(failed); 473 } 474 475 template< class ImplTraits> 476 ANTLR_INLINE typename Lexer<ImplTraits>::InputStreamType* Lexer<ImplTraits>::get_strstream() const 477 { 478 return this->get_input(); 479 } 480 481 template< class ImplTraits> 482 ANTLR_INLINE ANTLR_MARKER Lexer<ImplTraits>::index() const 483 { 484 return this->get_istream()->index(); 485 } 486 487 template< class ImplTraits> 488 ANTLR_INLINE void Lexer<ImplTraits>::seek(ANTLR_MARKER index) 489 { 490 this->get_istream()->seek(index); 491 } 492 493 template< class ImplTraits> 494 ANTLR_INLINE const typename Lexer<ImplTraits>::CommonTokenType* Lexer<ImplTraits>::EOF_Token() const 495 { 496 const CommonTokenType& eof_token = this->get_tokSource()->get_eofToken(); 497 return &eof_token; 498 } 499 500 template< class ImplTraits> 501 ANTLR_INLINE bool Lexer<ImplTraits>::hasException() const 502 { 503 return this->get_lexstate()->get_error(); 504 } 505 506 template< class ImplTraits> 507 ANTLR_INLINE typename Lexer<ImplTraits>::ExceptionBaseType* Lexer<ImplTraits>::get_exception() const 508 { 509 return this->get_lexstate()->get_exception(); 510 } 511 512 template< class ImplTraits> 513 ANTLR_INLINE void Lexer<ImplTraits>::constructEx() 514 { 515 this->get_rec()->exConstruct(); 516 } 517 518 template< class ImplTraits> 519 ANTLR_INLINE ANTLR_MARKER Lexer<ImplTraits>::mark() 520 { 521 return this->get_istream()->mark(); 522 } 523 524 template< class ImplTraits> 525 ANTLR_INLINE void Lexer<ImplTraits>::rewind(ANTLR_MARKER marker) 526 { 527 this->get_istream()->rewind(marker); 528 } 529 530 template< class ImplTraits> 531 ANTLR_INLINE void Lexer<ImplTraits>::rewindLast() 532 { 533 this->get_istream()->rewindLast(); 534 } 535 536 template< class ImplTraits> 537 ANTLR_INLINE void Lexer<ImplTraits>::memoize(ANTLR_MARKER ruleIndex, ANTLR_MARKER ruleParseStart) 538 { 539 this->get_rec()->memoize( ruleIndex, ruleParseStart ); 540 } 541 542 template< class ImplTraits> 543 ANTLR_INLINE bool Lexer<ImplTraits>::haveParsedRule(ANTLR_MARKER ruleIndex) 544 { 545 return this->get_rec()->alreadyParsedRule(ruleIndex); 546 } 547 548 template< class ImplTraits> 549 ANTLR_INLINE void Lexer<ImplTraits>::setText( const StringType& text ) 550 { 551 this->get_lexstate()->set_text(text); 552 } 553 554 template< class ImplTraits> 555 ANTLR_INLINE void Lexer<ImplTraits>::skip() 556 { 557 CommonTokenType& skipToken = this->get_tokSource()->get_skipToken(); 558 this->get_lexstate()->set_token( &skipToken ); 559 } 560 561 template< class ImplTraits> 562 ANTLR_INLINE typename Lexer<ImplTraits>::RuleMemoType* Lexer<ImplTraits>::getRuleMemo() const 563 { 564 return this->get_lexstate()->get_rulememo(); 565 } 566 567 template< class ImplTraits> 568 ANTLR_INLINE void Lexer<ImplTraits>::setRuleMemo(RuleMemoType* rulememo) 569 { 570 return this->get_lexstate()->set_rulememo(rulememo); 571 } 572 573 template< class ImplTraits> 574 ANTLR_INLINE typename Lexer<ImplTraits>::DebuggerType* Lexer<ImplTraits>::get_debugger() const 575 { 576 return this->get_rec()->get_debugger(); 577 } 578 579 template< class ImplTraits> 580 ANTLR_INLINE ANTLR_UINT32 Lexer<ImplTraits>::LA(ANTLR_INT32 i) 581 { 582 return this->get_istream()->_LA(i); 583 } 584 585 template< class ImplTraits> 586 ANTLR_INLINE void Lexer<ImplTraits>::consume() 587 { 588 return this->get_istream()->consume(); 589 } 590 591 ANTLR_END_NAMESPACE() 592 593
