1 /// \file 2 /// Default implementation of CommonTokenStream 3 /// 4 5 // [The "BSD licence"] 6 // Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC 7 // http://www.temporal-wave.com 8 // http://www.linkedin.com/in/jimidle 9 // 10 // All rights reserved. 11 // 12 // Redistribution and use in source and binary forms, with or without 13 // modification, are permitted provided that the following conditions 14 // are met: 15 // 1. Redistributions of source code must retain the above copyright 16 // notice, this list of conditions and the following disclaimer. 17 // 2. Redistributions in binary form must reproduce the above copyright 18 // notice, this list of conditions and the following disclaimer in the 19 // documentation and/or other materials provided with the distribution. 20 // 3. The name of the author may not be used to endorse or promote products 21 // derived from this software without specific prior written permission. 22 // 23 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26 // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27 // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28 // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32 // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 34 #include <antlr3tokenstream.h> 35 36 #ifdef ANTLR3_WINDOWS 37 #pragma warning( disable : 4100 ) 38 #endif 39 40 // COMMON_TOKEN_STREAM API 41 // 42 static void setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel); 43 static void discardTokenType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype); 44 static void discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard); 45 static pANTLR3_VECTOR getTokens (pANTLR3_COMMON_TOKEN_STREAM cts); 46 static pANTLR3_LIST getTokenRange (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop); 47 static pANTLR3_LIST getTokensSet (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types); 48 static pANTLR3_LIST getTokensList (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list); 49 static pANTLR3_LIST getTokensType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type); 50 static void reset (pANTLR3_COMMON_TOKEN_STREAM cts); 51 52 // TOKEN_STREAM API 53 // 54 static pANTLR3_COMMON_TOKEN tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k); 55 static pANTLR3_COMMON_TOKEN dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k); 56 static pANTLR3_COMMON_TOKEN get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i); 57 static pANTLR3_TOKEN_SOURCE getTokenSource (pANTLR3_TOKEN_STREAM ts); 58 static void setTokenSource (pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource); 59 static pANTLR3_STRING toString (pANTLR3_TOKEN_STREAM ts); 60 static pANTLR3_STRING toStringSS (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop); 61 static pANTLR3_STRING toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop); 62 static void setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger); 63 64 // INT STREAM API 65 // 66 static void consume (pANTLR3_INT_STREAM is); 67 static void dbgConsume (pANTLR3_INT_STREAM is); 68 static ANTLR3_UINT32 _LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i); 69 static ANTLR3_UINT32 dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i); 70 static ANTLR3_MARKER mark (pANTLR3_INT_STREAM is); 71 static ANTLR3_MARKER dbgMark (pANTLR3_INT_STREAM is); 72 static void release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark); 73 static ANTLR3_UINT32 size (pANTLR3_INT_STREAM is); 74 static ANTLR3_MARKER tindex (pANTLR3_INT_STREAM is); 75 static void rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker); 76 static void dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker); 77 static void rewindLast (pANTLR3_INT_STREAM is); 78 static void dbgRewindLast (pANTLR3_INT_STREAM is); 79 static void seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index); 80 static void dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index); 81 static pANTLR3_STRING getSourceName (pANTLR3_INT_STREAM is); 82 static void antlr3TokenStreamFree (pANTLR3_TOKEN_STREAM stream); 83 static void antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream); 84 85 // Helpers 86 // 87 static void fillBuffer (pANTLR3_COMMON_TOKEN_STREAM tokenStream); 88 static ANTLR3_UINT32 skipOffTokenChannels (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); 89 static ANTLR3_UINT32 skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); 90 static pANTLR3_COMMON_TOKEN LB (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); 91 92 ANTLR3_API pANTLR3_TOKEN_STREAM 93 antlr3TokenStreamNew() 94 { 95 pANTLR3_TOKEN_STREAM stream; 96 97 // Memory for the interface structure 98 // 99 stream = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM)); 100 101 if (stream == NULL) 102 { 103 return NULL; 104 } 105 106 // Install basic API 107 // 108 stream->free = antlr3TokenStreamFree; 109 110 111 return stream; 112 } 113 114 static void 115 antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream) 116 { 117 ANTLR3_FREE(stream); 118 } 119 120 static void 121 antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream) 122 { 123 // We only free up our subordinate interfaces if they belong 124 // to us, otherwise we let whoever owns them deal with them. 125 // 126 if (stream->tstream->super == stream) 127 { 128 if (stream->tstream->istream->super == stream->tstream) 129 { 130 stream->tstream->istream->free(stream->tstream->istream); 131 stream->tstream->istream = NULL; 132 } 133 stream->tstream->free(stream->tstream); 134 } 135 136 // Now we free our own resources 137 // 138 if (stream->tokens != NULL) 139 { 140 stream->tokens->free(stream->tokens); 141 stream->tokens = NULL; 142 } 143 if (stream->discardSet != NULL) 144 { 145 stream->discardSet->free(stream->discardSet); 146 stream->discardSet = NULL; 147 } 148 if (stream->channelOverrides != NULL) 149 { 150 stream->channelOverrides->free(stream->channelOverrides); 151 stream->channelOverrides = NULL; 152 } 153 154 // Free our memory now 155 // 156 ANTLR3_FREE(stream); 157 } 158 159 // Reset a token stream so it can be used again and can reuse it's 160 // resources. 161 // 162 static void 163 reset (pANTLR3_COMMON_TOKEN_STREAM cts) 164 { 165 166 // Free any resources that ar most like specifc to the 167 // run we just did. 168 // 169 if (cts->discardSet != NULL) 170 { 171 cts->discardSet->free(cts->discardSet); 172 cts->discardSet = NULL; 173 } 174 if (cts->channelOverrides != NULL) 175 { 176 cts->channelOverrides->free(cts->channelOverrides); 177 cts->channelOverrides = NULL; 178 } 179 180 // Now, if there were any existing tokens in the stream, 181 // then we just reset the vector count so that it starts 182 // again. We must traverse the entries unfortunately as 183 // there may be free pointers for custom token types and 184 // so on. However that is just a quick NULL check on the 185 // vector entries. 186 // 187 if (cts->tokens != NULL) 188 { 189 cts->tokens->clear(cts->tokens); 190 } 191 else 192 { 193 /* Install the token tracking tables 194 */ 195 cts->tokens = antlr3VectorNew(0); 196 } 197 198 // Reset to defaults 199 // 200 cts->discardOffChannel = ANTLR3_FALSE; 201 cts->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL; 202 cts->p = -1; 203 } 204 205 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM 206 antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger) 207 { 208 pANTLR3_COMMON_TOKEN_STREAM stream; 209 210 // Create a standard token stream 211 // 212 stream = antlr3CommonTokenStreamSourceNew(hint, source); 213 214 // Install the debugger object 215 // 216 stream->tstream->debugger = debugger; 217 218 // Override standard token stream methods with debugging versions 219 // 220 stream->tstream->initialStreamState = ANTLR3_FALSE; 221 222 stream->tstream->_LT = dbgTokLT; 223 224 stream->tstream->istream->consume = dbgConsume; 225 stream->tstream->istream->_LA = dbgLA; 226 stream->tstream->istream->mark = dbgMark; 227 stream->tstream->istream->rewind = dbgRewindStream; 228 stream->tstream->istream->rewindLast = dbgRewindLast; 229 stream->tstream->istream->seek = dbgSeek; 230 231 return stream; 232 } 233 234 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM 235 antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source) 236 { 237 pANTLR3_COMMON_TOKEN_STREAM stream; 238 239 stream = antlr3CommonTokenStreamNew(hint); 240 241 stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL; 242 243 stream->channelOverrides = NULL; 244 stream->discardSet = NULL; 245 stream->discardOffChannel = ANTLR3_FALSE; 246 247 stream->tstream->setTokenSource(stream->tstream, source); 248 249 stream->free = antlr3CTSFree; 250 return stream; 251 } 252 253 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM 254 antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint) 255 { 256 pANTLR3_COMMON_TOKEN_STREAM stream; 257 258 /* Memory for the interface structure 259 */ 260 stream = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM)); 261 262 if (stream == NULL) 263 { 264 return NULL; 265 } 266 267 /* Create space for the token stream interface 268 */ 269 stream->tstream = antlr3TokenStreamNew(); 270 stream->tstream->super = stream; 271 272 /* Create space for the INT_STREAM interfacce 273 */ 274 stream->tstream->istream = antlr3IntStreamNew(); 275 stream->tstream->istream->super = (stream->tstream); 276 stream->tstream->istream->type = ANTLR3_TOKENSTREAM; 277 278 /* Install the token tracking tables 279 */ 280 stream->tokens = antlr3VectorNew(0); 281 282 /* Defaults 283 */ 284 stream->p = -1; 285 286 /* Install the common token stream API 287 */ 288 stream->setTokenTypeChannel = setTokenTypeChannel; 289 stream->discardTokenType = discardTokenType; 290 stream->discardOffChannelToks = discardOffChannel; 291 stream->getTokens = getTokens; 292 stream->getTokenRange = getTokenRange; 293 stream->getTokensSet = getTokensSet; 294 stream->getTokensList = getTokensList; 295 stream->getTokensType = getTokensType; 296 stream->reset = reset; 297 298 /* Install the token stream API 299 */ 300 stream->tstream->_LT = tokLT; 301 stream->tstream->get = get; 302 stream->tstream->getTokenSource = getTokenSource; 303 stream->tstream->setTokenSource = setTokenSource; 304 stream->tstream->toString = toString; 305 stream->tstream->toStringSS = toStringSS; 306 stream->tstream->toStringTT = toStringTT; 307 stream->tstream->setDebugListener = setDebugListener; 308 309 /* Install INT_STREAM interface 310 */ 311 stream->tstream->istream->_LA = _LA; 312 stream->tstream->istream->mark = mark; 313 stream->tstream->istream->release = release; 314 stream->tstream->istream->size = size; 315 stream->tstream->istream->index = tindex; 316 stream->tstream->istream->rewind = rewindStream; 317 stream->tstream->istream->rewindLast= rewindLast; 318 stream->tstream->istream->seek = seek; 319 stream->tstream->istream->consume = consume; 320 stream->tstream->istream->getSourceName = getSourceName; 321 322 return stream; 323 } 324 325 // Install a debug listener adn switch to debug mode methods 326 // 327 static void 328 setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger) 329 { 330 // Install the debugger object 331 // 332 ts->debugger = debugger; 333 334 // Override standard token stream methods with debugging versions 335 // 336 ts->initialStreamState = ANTLR3_FALSE; 337 338 ts->_LT = dbgTokLT; 339 340 ts->istream->consume = dbgConsume; 341 ts->istream->_LA = dbgLA; 342 ts->istream->mark = dbgMark; 343 ts->istream->rewind = dbgRewindStream; 344 ts->istream->rewindLast = dbgRewindLast; 345 ts->istream->seek = dbgSeek; 346 } 347 348 /** Get the ith token from the current position 1..n where k=1 is the 349 * first symbol of lookahead. 350 */ 351 static pANTLR3_COMMON_TOKEN 352 tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k) 353 { 354 ANTLR3_INT32 i; 355 ANTLR3_INT32 n; 356 pANTLR3_COMMON_TOKEN_STREAM cts; 357 358 cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; 359 360 if (k < 0) 361 { 362 return LB(cts, -k); 363 } 364 365 if (cts->p == -1) 366 { 367 fillBuffer(cts); 368 } 369 370 // Here we used to check for k == 0 and return 0, but this seems 371 // a superfluous check to me. LT(k=0) is therefore just undefined 372 // and we won't waste the clock cycles on the check 373 // 374 375 if ((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize) 376 { 377 pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken); 378 379 teof->setStartIndex (teof, ts->istream->index (ts->istream)); 380 teof->setStopIndex (teof, ts->istream->index (ts->istream)); 381 return teof; 382 } 383 384 i = cts->p; 385 n = 1; 386 387 /* Need to find k good tokens, skipping ones that are off channel 388 */ 389 while ( n < k) 390 { 391 /* Skip off-channel tokens */ 392 i = skipOffTokenChannels(cts, i+1); /* leave p on valid token */ 393 n++; 394 } 395 if ( (ANTLR3_UINT32) i >= ts->istream->cachedSize) 396 { 397 pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken); 398 399 teof->setStartIndex (teof, ts->istream->index(ts->istream)); 400 teof->setStopIndex (teof, ts->istream->index(ts->istream)); 401 return teof; 402 } 403 404 // Here the token must be in the input vector. Rather then incur 405 // function call penalty, we just return the pointer directly 406 // from the vector 407 // 408 return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element; 409 //return (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i); 410 } 411 412 /// Debug only method to flag consumption of initial off-channel 413 /// tokens in the input stream 414 /// 415 static void 416 consumeInitialHiddenTokens(pANTLR3_INT_STREAM is) 417 { 418 ANTLR3_MARKER first; 419 ANTLR3_INT32 i; 420 pANTLR3_TOKEN_STREAM ts; 421 422 ts = (pANTLR3_TOKEN_STREAM) is->super; 423 first = is->index(is); 424 425 for (i=0; i<first; i++) 426 { 427 ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i)); 428 } 429 430 ts->initialStreamState = ANTLR3_FALSE; 431 432 } 433 434 /// As per the normal tokLT but sends information to the debugger 435 /// 436 static pANTLR3_COMMON_TOKEN 437 dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k) 438 { 439 if (ts->initialStreamState == ANTLR3_TRUE) 440 { 441 consumeInitialHiddenTokens(ts->istream); 442 } 443 return tokLT(ts, k); 444 } 445 446 #ifdef ANTLR3_WINDOWS 447 /* When fully optimized VC7 complains about non reachable code. 448 * Not yet sure if this is an optimizer bug, or a bug in the flow analysis 449 */ 450 #pragma warning( disable : 4702 ) 451 #endif 452 453 static pANTLR3_COMMON_TOKEN 454 LB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k) 455 { 456 ANTLR3_INT32 i; 457 ANTLR3_INT32 n; 458 459 if (cts->p == -1) 460 { 461 fillBuffer(cts); 462 } 463 if (k == 0) 464 { 465 return NULL; 466 } 467 if ((cts->p - k) < 0) 468 { 469 return NULL; 470 } 471 472 i = cts->p; 473 n = 1; 474 475 /* Need to find k good tokens, going backwards, skipping ones that are off channel 476 */ 477 while (n <= (ANTLR3_INT32) k) 478 { 479 /* Skip off-channel tokens 480 */ 481 482 i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token */ 483 n++; 484 } 485 if (i < 0) 486 { 487 return NULL; 488 } 489 // Here the token must be in the input vector. Rather then incut 490 // function call penalty, we jsut return the pointer directly 491 // from the vector 492 // 493 return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element; 494 } 495 496 static pANTLR3_COMMON_TOKEN 497 get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i) 498 { 499 pANTLR3_COMMON_TOKEN_STREAM cts; 500 501 cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; 502 503 return (pANTLR3_COMMON_TOKEN)(cts->tokens->get(cts->tokens, i)); /* Token index is zero based but vectors are 1 based */ 504 } 505 506 static pANTLR3_TOKEN_SOURCE 507 getTokenSource (pANTLR3_TOKEN_STREAM ts) 508 { 509 return ts->tokenSource; 510 } 511 512 static void 513 setTokenSource ( pANTLR3_TOKEN_STREAM ts, 514 pANTLR3_TOKEN_SOURCE tokenSource) 515 { 516 ts->tokenSource = tokenSource; 517 } 518 519 static pANTLR3_STRING 520 toString (pANTLR3_TOKEN_STREAM ts) 521 { 522 pANTLR3_COMMON_TOKEN_STREAM cts; 523 524 cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; 525 526 if (cts->p == -1) 527 { 528 fillBuffer(cts); 529 } 530 531 return ts->toStringSS(ts, 0, ts->istream->size(ts->istream)); 532 } 533 534 static pANTLR3_STRING 535 toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop) 536 { 537 pANTLR3_STRING string; 538 pANTLR3_TOKEN_SOURCE tsource; 539 pANTLR3_COMMON_TOKEN tok; 540 ANTLR3_UINT32 i; 541 pANTLR3_COMMON_TOKEN_STREAM cts; 542 543 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 544 545 if (cts->p == -1) 546 { 547 fillBuffer(cts); 548 } 549 if (stop >= ts->istream->size(ts->istream)) 550 { 551 stop = ts->istream->size(ts->istream) - 1; 552 } 553 554 /* Who is giving us these tokens? 555 */ 556 tsource = ts->getTokenSource(ts); 557 558 if (tsource != NULL && cts->tokens != NULL) 559 { 560 /* Finally, let's get a string 561 */ 562 string = tsource->strFactory->newRaw(tsource->strFactory); 563 564 for (i = start; i <= stop; i++) 565 { 566 tok = ts->get(ts, i); 567 if (tok != NULL) 568 { 569 string->appendS(string, tok->getText(tok)); 570 } 571 } 572 573 return string; 574 } 575 return NULL; 576 577 } 578 579 static pANTLR3_STRING 580 toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop) 581 { 582 if (start != NULL && stop != NULL) 583 { 584 return ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop)); 585 } 586 else 587 { 588 return NULL; 589 } 590 } 591 592 /** Move the input pointer to the next incoming token. The stream 593 * must become active with LT(1) available. consume() simply 594 * moves the input pointer so that LT(1) points at the next 595 * input symbol. Consume at least one token. 596 * 597 * Walk past any token not on the channel the parser is listening to. 598 */ 599 static void 600 consume (pANTLR3_INT_STREAM is) 601 { 602 pANTLR3_COMMON_TOKEN_STREAM cts; 603 pANTLR3_TOKEN_STREAM ts; 604 605 ts = (pANTLR3_TOKEN_STREAM) is->super; 606 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 607 608 if ((ANTLR3_UINT32)cts->p < cts->tokens->count) 609 { 610 cts->p++; 611 cts->p = skipOffTokenChannels(cts, cts->p); 612 } 613 } 614 615 616 /// As per ordinary consume but notifies the debugger about hidden 617 /// tokens and so on. 618 /// 619 static void 620 dbgConsume (pANTLR3_INT_STREAM is) 621 { 622 pANTLR3_TOKEN_STREAM ts; 623 ANTLR3_MARKER a; 624 ANTLR3_MARKER b; 625 pANTLR3_COMMON_TOKEN t; 626 627 ts = (pANTLR3_TOKEN_STREAM) is->super; 628 629 if (ts->initialStreamState == ANTLR3_TRUE) 630 { 631 consumeInitialHiddenTokens(is); 632 } 633 634 a = is->index(is); // Where are we right now? 635 t = ts->_LT(ts, 1); // Current token from stream 636 637 consume(is); // Standard consumer 638 639 b = is->index(is); // Where are we after consuming 1 on channel token? 640 641 ts->debugger->consumeToken(ts->debugger, t); // Tell the debugger that we consumed the first token 642 643 if (b>a+1) 644 { 645 // The standard consume caused the index to advance by more than 1, 646 // which can only happen if it skipped some off-channel tokens. 647 // we need to tell the debugger about those tokens. 648 // 649 ANTLR3_MARKER i; 650 651 for (i = a+1; i<b; i++) 652 { 653 ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i)); 654 } 655 656 } 657 } 658 659 /** A simple filter mechanism whereby you can tell this token stream 660 * to force all tokens of type ttype to be on channel. For example, 661 * when interpreting, we cannot execute actions so we need to tell 662 * the stream to force all WS and NEWLINE to be a different, ignored, 663 * channel. 664 */ 665 static void 666 setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel) 667 { 668 if (tokenStream->channelOverrides == NULL) 669 { 670 tokenStream->channelOverrides = antlr3ListNew(10); 671 } 672 673 /* We add one to the channel so we can distinguish NULL as being no entry in the 674 * table for a particular token type. 675 */ 676 tokenStream->channelOverrides->put(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL); 677 } 678 679 static void 680 discardTokenType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype) 681 { 682 if (tokenStream->discardSet == NULL) 683 { 684 tokenStream->discardSet = antlr3ListNew(31); 685 } 686 687 /* We add one to the channel so we can distinguish NULL as being no entry in the 688 * table for a particular token type. We could use bitsets for this I suppose too. 689 */ 690 tokenStream->discardSet->put(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL); 691 } 692 693 static void 694 discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard) 695 { 696 tokenStream->discardOffChannel = discard; 697 } 698 699 static pANTLR3_VECTOR 700 getTokens (pANTLR3_COMMON_TOKEN_STREAM tokenStream) 701 { 702 if (tokenStream->p == -1) 703 { 704 fillBuffer(tokenStream); 705 } 706 707 return tokenStream->tokens; 708 } 709 710 static pANTLR3_LIST 711 getTokenRange (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop) 712 { 713 return tokenStream->getTokensSet(tokenStream, start, stop, NULL); 714 } 715 /** Given a start and stop index, return a List of all tokens in 716 * the token type BitSet. Return null if no tokens were found. This 717 * method looks at both on and off channel tokens. 718 */ 719 static pANTLR3_LIST 720 getTokensSet (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types) 721 { 722 pANTLR3_LIST filteredList; 723 ANTLR3_UINT32 i; 724 ANTLR3_UINT32 n; 725 pANTLR3_COMMON_TOKEN tok; 726 727 if (tokenStream->p == -1) 728 { 729 fillBuffer(tokenStream); 730 } 731 if (stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream)) 732 { 733 stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream); 734 } 735 if (start > stop) 736 { 737 return NULL; 738 } 739 740 /* We have the range set, now we need to iterate through the 741 * installed tokens and create a new list with just the ones we want 742 * in it. We are just moving pointers about really. 743 */ 744 filteredList = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream)); 745 746 for (i = start, n = 0; i<= stop; i++) 747 { 748 tok = tokenStream->tstream->get(tokenStream->tstream, i); 749 750 if ( types == NULL 751 || types->isMember(types, tok->getType(tok) == ANTLR3_TRUE) 752 ) 753 { 754 filteredList->put(filteredList, n++, (void *)tok, NULL); 755 } 756 } 757 758 /* Did we get any then? 759 */ 760 if (filteredList->size(filteredList) == 0) 761 { 762 filteredList->free(filteredList); 763 filteredList = NULL; 764 } 765 766 return filteredList; 767 } 768 769 static pANTLR3_LIST 770 getTokensList (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list) 771 { 772 pANTLR3_BITSET bitSet; 773 pANTLR3_LIST newlist; 774 775 bitSet = antlr3BitsetList(list->table); 776 777 newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet); 778 779 bitSet->free(bitSet); 780 781 return newlist; 782 783 } 784 785 static pANTLR3_LIST 786 getTokensType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type) 787 { 788 pANTLR3_BITSET bitSet; 789 pANTLR3_LIST newlist; 790 791 bitSet = antlr3BitsetOf(type, -1); 792 newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet); 793 794 bitSet->free(bitSet); 795 796 return newlist; 797 } 798 799 static ANTLR3_UINT32 800 _LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i) 801 { 802 pANTLR3_TOKEN_STREAM ts; 803 pANTLR3_COMMON_TOKEN tok; 804 805 ts = (pANTLR3_TOKEN_STREAM) is->super; 806 807 tok = ts->_LT(ts, i); 808 809 if (tok != NULL) 810 { 811 return tok->getType(tok); 812 } 813 else 814 { 815 return ANTLR3_TOKEN_INVALID; 816 } 817 } 818 819 /// As per _LA() but for debug mode. 820 /// 821 static ANTLR3_UINT32 822 dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i) 823 { 824 pANTLR3_TOKEN_STREAM ts; 825 826 ts = (pANTLR3_TOKEN_STREAM) is->super; 827 828 if (ts->initialStreamState == ANTLR3_TRUE) 829 { 830 consumeInitialHiddenTokens(is); 831 } 832 ts->debugger->LT(ts->debugger, i, tokLT(ts, i)); 833 return _LA(is, i); 834 } 835 836 static ANTLR3_MARKER 837 mark (pANTLR3_INT_STREAM is) 838 { 839 is->lastMarker = is->index(is); 840 return is->lastMarker; 841 } 842 843 /// As per mark() but with a call to tell the debugger we are doing this 844 /// 845 static ANTLR3_MARKER 846 dbgMark (pANTLR3_INT_STREAM is) 847 { 848 pANTLR3_TOKEN_STREAM ts; 849 850 ts = (pANTLR3_TOKEN_STREAM) is->super; 851 852 is->lastMarker = is->index(is); 853 ts->debugger->mark(ts->debugger, is->lastMarker); 854 855 return is->lastMarker; 856 } 857 858 static void 859 release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark) 860 { 861 return; 862 } 863 864 static ANTLR3_UINT32 865 size (pANTLR3_INT_STREAM is) 866 { 867 pANTLR3_COMMON_TOKEN_STREAM cts; 868 pANTLR3_TOKEN_STREAM ts; 869 870 if (is->cachedSize > 0) 871 { 872 return is->cachedSize; 873 } 874 ts = (pANTLR3_TOKEN_STREAM) is->super; 875 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 876 877 is->cachedSize = cts->tokens->count; 878 return is->cachedSize; 879 } 880 881 static ANTLR3_MARKER 882 tindex (pANTLR3_INT_STREAM is) 883 { 884 pANTLR3_COMMON_TOKEN_STREAM cts; 885 pANTLR3_TOKEN_STREAM ts; 886 887 ts = (pANTLR3_TOKEN_STREAM) is->super; 888 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 889 890 return cts->p; 891 } 892 893 static void 894 dbgRewindLast (pANTLR3_INT_STREAM is) 895 { 896 pANTLR3_TOKEN_STREAM ts; 897 898 ts = (pANTLR3_TOKEN_STREAM) is->super; 899 900 ts->debugger->rewindLast(ts->debugger); 901 902 is->rewind(is, is->lastMarker); 903 } 904 static void 905 rewindLast (pANTLR3_INT_STREAM is) 906 { 907 is->rewind(is, is->lastMarker); 908 } 909 static void 910 rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker) 911 { 912 is->seek(is, (ANTLR3_UINT32)(marker)); 913 } 914 static void 915 dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker) 916 { 917 pANTLR3_TOKEN_STREAM ts; 918 919 ts = (pANTLR3_TOKEN_STREAM) is->super; 920 921 ts->debugger->rewind(ts->debugger, marker); 922 923 is->seek(is, (ANTLR3_UINT32)(marker)); 924 } 925 926 static void 927 seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index) 928 { 929 pANTLR3_COMMON_TOKEN_STREAM cts; 930 pANTLR3_TOKEN_STREAM ts; 931 932 ts = (pANTLR3_TOKEN_STREAM) is->super; 933 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 934 935 cts->p = (ANTLR3_UINT32)index; 936 } 937 static void 938 dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index) 939 { 940 // TODO: Implement seek in debugger when Ter adds it to Java 941 // 942 seek(is, index); 943 } 944 ANTLR3_API void 945 fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream) 946 { 947 fillBuffer(tokenStream); 948 } 949 static void 950 fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) { 951 ANTLR3_UINT32 index; 952 pANTLR3_COMMON_TOKEN tok; 953 ANTLR3_BOOLEAN discard; 954 void * channelI; 955 956 /* Start at index 0 of course 957 */ 958 index = 0; 959 960 /* Pick out the next token from the token source 961 * Remember we just get a pointer (reference if you like) here 962 * and so if we store it anywhere, we don't set any pointers to auto free it. 963 */ 964 tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource); 965 966 while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF) 967 { 968 discard = ANTLR3_FALSE; /* Assume we are not discarding */ 969 970 /* I employ a bit of a trick, or perhaps hack here. Rather than 971 * store a pointer to a structure in the override map and discard set 972 * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0 973 * we can distinguish "not being there" from "being channel or type 0" 974 */ 975 976 if (tokenStream->discardSet != NULL 977 && tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL) 978 { 979 discard = ANTLR3_TRUE; 980 } 981 else if ( tokenStream->discardOffChannel == ANTLR3_TRUE 982 && tok->getChannel(tok) != tokenStream->channel 983 ) 984 { 985 discard = ANTLR3_TRUE; 986 } 987 else if (tokenStream->channelOverrides != NULL) 988 { 989 /* See if this type is in the override map 990 */ 991 channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1); 992 993 if (channelI != NULL) 994 { 995 /* Override found 996 */ 997 tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1); 998 } 999 } 1000 1001 /* If not discarding it, add it to the list at the current index 1002 */ 1003 if (discard == ANTLR3_FALSE) 1004 { 1005 /* Add it, indicating that we will delete it and the table should not 1006 */ 1007 tok->setTokenIndex(tok, index); 1008 tokenStream->p++; 1009 tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL); 1010 index++; 1011 } 1012 1013 tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource); 1014 } 1015 1016 /* Cache the size so we don't keep doing indirect method calls. We do this as 1017 * early as possible so that anything after this may utilize the cached value. 1018 */ 1019 tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count; 1020 1021 /* Set the consume pointer to the first token that is on our channel 1022 */ 1023 tokenStream->p = 0; 1024 tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p); 1025 1026 } 1027 1028 /// Given a starting index, return the index of the first on-channel 1029 /// token. 1030 /// 1031 static ANTLR3_UINT32 1032 skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) { 1033 ANTLR3_INT32 n; 1034 pANTLR3_COMMON_TOKEN tok; 1035 1036 n = tokenStream->tstream->istream->cachedSize; 1037 1038 while (i < n) 1039 { 1040 tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element; 1041 1042 if (tok->channel!= tokenStream->channel) 1043 { 1044 i++; 1045 } 1046 else 1047 { 1048 return i; 1049 } 1050 } 1051 return i; 1052 } 1053 1054 static ANTLR3_UINT32 1055 skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x) 1056 { 1057 pANTLR3_COMMON_TOKEN tok; 1058 1059 while (x >= 0) 1060 { 1061 tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element; 1062 1063 if ((tok->channel != tokenStream->channel)) 1064 { 1065 x--; 1066 } 1067 else 1068 { 1069 return x; 1070 } 1071 } 1072 return x; 1073 } 1074 1075 /// Return a string that represents the name assoicated with the input source 1076 /// 1077 /// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream. 1078 /// 1079 /// /returns 1080 /// /implements ANTLR3_INT_STREAM_struct::getSourceName() 1081 /// 1082 static pANTLR3_STRING 1083 getSourceName (pANTLR3_INT_STREAM is) 1084 { 1085 // Slightly convoluted as we must trace back to the lexer's input source 1086 // via the token source. The streamName that is here is not initialized 1087 // because this is a token stream, not a file or string stream, which are the 1088 // only things that have a context for a source name. 1089 // 1090 return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName; 1091 } 1092
