Lines Matching refs:lexer
3 * Base implementation of an antlr 3 lexer.
5 * An ANTLR3 lexer implements a base recongizer, a token source and
6 * a lexer interface. It constructs a base recognizer with default
42 static void mTokens (pANTLR3_LEXER lexer);
43 static void setCharStream (pANTLR3_LEXER lexer, pANTLR3_INPUT_STREAM input);
44 static void pushCharStream (pANTLR3_LEXER lexer, pANTLR3_INPUT_STREAM input);
45 static void popCharStream (pANTLR3_LEXER lexer);
47 static void emitNew (pANTLR3_LEXER lexer, pANTLR3_COMMON_TOKEN token);
48 static pANTLR3_COMMON_TOKEN emit (pANTLR3_LEXER lexer);
49 static ANTLR3_BOOLEAN matchs (pANTLR3_LEXER lexer, ANTLR3_UCHAR * string);
50 static ANTLR3_BOOLEAN matchc (pANTLR3_LEXER lexer, ANTLR3_UCHAR c);
51 static ANTLR3_BOOLEAN matchRange (pANTLR3_LEXER lexer, ANTLR3_UCHAR low, ANTLR3_UCHAR high);
52 static void matchAny (pANTLR3_LEXER lexer);
53 static void recover (pANTLR3_LEXER lexer);
54 static ANTLR3_UINT32 getLine (pANTLR3_LEXER lexer);
55 static ANTLR3_MARKER getCharIndex (pANTLR3_LEXER lexer);
56 static ANTLR3_UINT32 getCharPositionInLine (pANTLR3_LEXER lexer);
57 static pANTLR3_STRING getText (pANTLR3_LEXER lexer);
68 static void freeLexer (pANTLR3_LEXER lexer);
74 pANTLR3_LEXER lexer;
79 lexer = (pANTLR3_LEXER) ANTLR3_MALLOC(sizeof(ANTLR3_LEXER));
81 if (lexer == NULL)
88 lexer->rec = antlr3BaseRecognizerNew(ANTLR3_TYPE_LEXER, sizeHint, state);
90 if (lexer->rec == NULL)
92 lexer->free(lexer);
95 lexer->rec->super = lexer;
97 lexer->rec->displayRecognitionError = displayRecognitionError;
98 lexer->rec->reportError = reportError;
99 lexer->rec->reset = reset;
100 lexer->rec->getCurrentInputSymbol = getCurrentInputSymbol;
101 lexer->rec->getMissingSymbol = getMissingSymbol;
105 if (lexer->rec->state->tokSource == NULL)
107 lexer->rec->state->tokSource = (pANTLR3_TOKEN_SOURCE)ANTLR3_CALLOC(1, sizeof(ANTLR3_TOKEN_SOURCE));
109 if (lexer->rec->state->tokSource == NULL)
111 lexer->rec->free(lexer->rec);
112 lexer->free(lexer);
116 lexer->rec->state->tokSource->super = lexer;
121 lexer->rec->state->tokSource->nextToken = nextToken;
122 lexer->rec->state->tokSource->strFactory = NULL;
124 lexer->rec->state->tokFactory = NULL;
127 /* Install the lexer API
129 lexer->setCharStream = setCharStream;
130 lexer->mTokens = (void (*)(void *))(mTokens);
131 lexer->setCharStream = setCharStream;
132 lexer->pushCharStream = pushCharStream;
133 lexer->popCharStream = popCharStream;
134 lexer->emit = emit;
135 lexer->emitNew = emitNew;
136 lexer->matchs = matchs;
137 lexer->matchc = matchc;
138 lexer->matchRange = matchRange;
139 lexer->matchAny = matchAny;
140 lexer->recover = recover;
141 lexer->getLine = getLine;
142 lexer->getCharIndex = getCharIndex;
143 lexer->getCharPositionInLine = getCharPositionInLine;
144 lexer->getText = getText;
145 lexer->free = freeLexer;
149 specialT = &(lexer->rec->state->tokSource->eofToken);
162 specialT = &(lexer->rec->state->tokSource->skipToken);
171 return lexer;
177 pANTLR3_LEXER lexer;
179 lexer = rec->super;
181 lexer->rec->state->token = NULL;
182 lexer->rec->state->type = ANTLR3_TOKEN_INVALID;
183 lexer->rec->state->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
184 lexer->rec->state->tokenStartCharIndex = -1;
185 lexer->rec->state->tokenStartCharPositionInLine = -1;
186 lexer->rec->state->tokenStartLine = -1;
188 lexer->rec->state->text = NULL;
193 if (lexer->rec->state->tokFactory != NULL)
195 lexer->rec->state->tokFactory->reset(lexer->rec->state->tokFactory);
204 /// Points to the implementation of a token source. The lexer is
219 pANTLR3_LEXER lexer;
224 lexer = (pANTLR3_LEXER)(toksource->super);
225 state = lexer->rec->state;
226 input = lexer->input;
263 teof->setStartIndex (teof, lexer->getCharIndex(lexer));
264 teof->setStopIndex (teof, lexer->getCharIndex(lexer));
265 teof->setLine (teof, lexer->getLine(lexer));
274 // Call the generated lexer, see if it can get a new token together.
276 lexer->mTokens(lexer->ctx);
283 lexer->rec->reportError(lexer->rec);
284 lexer->recover(lexer);
292 emit(lexer);
312 * Default implementation of the nextToken() call for a lexer.
315 * Points to the implementation of a token source. The lexer is
353 pANTLR3_LEXER lexer;
355 lexer = (pANTLR3_LEXER)(toksource->super);
357 if (lexer->rec->state->streams != NULL && lexer->rec->state->streams->size(lexer->rec->state->streams) > 0)
363 lexer->popCharStream(lexer);
386 pANTLR3_LEXER lexer;
388 // Create a basic lexer first
390 lexer = antlr3LexerNew(sizeHint, state);
392 if (lexer != NULL)
394 // Install the input stream and reset the lexer
396 setCharStream(lexer, input);
399 return lexer;
402 static void mTokens (pANTLR3_LEXER lexer)
404 if (lexer) // Fool compiler, avoid pragmas
406 ANTLR3_FPRINTF(stderr, "lexer->mTokens(): Error: No lexer rules were added to the lexer yet!\n");
424 /** Default lexer error handler (works for 8 bit streams only!!!)
429 pANTLR3_LEXER lexer;
433 lexer = (pANTLR3_LEXER)(recognizer->super);
434 ex = lexer->rec->state->exception;
449 ANTLR3_FPRINTF(stderr, ": lexer error %d :\n\t%s at offset %d, ",
457 width = ANTLR3_UINT32_CAST(( (pANTLR3_UINT8)(lexer->input->data) + (lexer->input->size(lexer->input) )) - (pANTLR3_UINT8)(ex->index));
473 ANTLR3_FPRINTF(stderr, "(end of input).\n\t This indicates a poorly specified lexer RULE\n\t or unterminated input element such as: \"STRING[\"]\n");
474 ANTLR3_FPRINTF(stderr, "\t The lexer was matching from line %d, offset %d, which\n\t ",
475 (ANTLR3_UINT32)(lexer->rec->state->tokenStartLine),
476 (ANTLR3_UINT32)(lexer->rec->state->tokenStartCharPositionInLine)
478 width = ANTLR3_UINT32_CAST(((pANTLR3_UINT8)(lexer->input->data)+(lexer->input->size(lexer->input))) - (pANTLR3_UINT8)(lexer->rec->state->tokenStartCharIndex));
482 ANTLR3_FPRINTF(stderr, "looks like this:\n\t\t%.*s\n", width > 20 ? 20 : width ,(pANTLR3_UINT8)(lexer->rec->state->tokenStartCharIndex));
486 ANTLR3_FPRINTF(stderr, "is also the end of the line, so you must check your lexer rules\n");
492 static void setCharStream (pANTLR3_LEXER lexer, pANTLR3_INPUT_STREAM input)
496 lexer->input = input;
498 /* We may need a token factory for the lexer; we don't destroy any existing factory
499 * until the lexer is destroyed, as people may still be using the tokens it produced.
503 if (lexer->rec->state->tokFactory == NULL)
505 lexer->rec->state->tokFactory = antlr3TokenFactoryNew(input);
510 * at the start of a new lexer, then we must tell the tokenFactory
515 lexer->rec->state->tokFactory->setInputStream(lexer->rec->state->tokFactory, input);
521 if (lexer->rec->state->tokSource->strFactory == NULL)
523 lexer->rec->state->tokSource->strFactory = input->strFactory;
528 if (lexer->rec->state->tokSource->eofToken.strFactory == NULL)
530 lexer->rec->state->tokSource->eofToken.strFactory = input->strFactory;
534 /* This is a lexer, install the appropriate exception creator
536 lexer->rec->exConstruct = antlr3RecognitionExceptionNew;
540 lexer->rec->state->token = NULL;
541 lexer->rec->state->text = NULL;
542 lexer->rec->state->tokenStartCharIndex = -1;
546 lexer->rec->state->tokSource->fileName = input->fileName;
553 * \param lexer
554 * Pointer to the lexer instance to switch input streams for.
564 pushCharStream (pANTLR3_LEXER lexer, pANTLR3_INPUT_STREAM input)
568 if (lexer->rec->state->streams == NULL)
573 lexer->rec->state->streams = antlr3StackNew(0);
575 if (lexer->rec->state->streams == NULL)
587 lexer->input->istream->mark(lexer->input->istream);
588 lexer->rec->state->streams->push(lexer->rec->state->streams, lexer->input, NULL);
592 lexer->setCharStream(lexer, input);
600 * \param lexer
601 * Description of parameter lexer.
610 popCharStream (pANTLR3_LEXER lexer)
617 if (lexer->rec->state->streams != NULL && lexer->rec->state->streams->size(lexer->rec->state->streams) > 0)
625 input = (pANTLR3_INPUT_STREAM)(lexer->rec->state->streams->top);
626 lexer->rec->state->streams->pop(lexer->rec->state->streams);
630 lexer->setCharStream(lexer, input);
631 lexer->input->istream->rewindLast(lexer->input->istream);
636 static void emitNew (pANTLR3_LEXER lexer, pANTLR3_COMMON_TOKEN token)
638 lexer->rec->state->token = token; /* Voila! */
642 emit (pANTLR3_LEXER lexer)
651 token = lexer->rec->state->tokFactory->newToken(lexer->rec->state->tokFactory);
657 token->type = lexer->rec->state->type;
658 token->channel = lexer->rec->state->channel;
659 token->start = lexer->rec->state->tokenStartCharIndex;
660 token->stop = lexer->getCharIndex(lexer) - 1;
661 token->line = lexer->rec->state->tokenStartLine;
662 token->charPosition = lexer->rec->state->tokenStartCharPositionInLine;
664 if (lexer->rec->state->text != NULL)
667 token->tokText.text = lexer->rec->state->text;
673 token->lineStart = lexer->input->currentLine;
674 token->user1 = lexer->rec->state->user1;
675 token->user2 = lexer->rec->state->user2;
676 token->user3 = lexer->rec->state->user3;
677 token->custom = lexer->rec->state->custom;
679 lexer->rec->state->token = token;
685 * Free the resources allocated by a lexer
688 freeLexer (pANTLR3_LEXER lexer)
690 // This may have ben a delegate or delegator lexer, in which case the
694 if (lexer->rec->state != NULL)
696 if (lexer->rec->state->streams != NULL)
698 lexer->rec->state->streams->free(lexer->rec->state->streams);
700 if (lexer->rec->state->tokFactory != NULL)
702 lexer->rec->state->tokFactory->close(lexer->rec->state->tokFactory);
703 lexer->rec->state->tokFactory = NULL;
705 if (lexer->rec->state->tokSource != NULL)
707 ANTLR3_FREE(lexer->rec->state->tokSource);
708 lexer->rec->state->tokSource = NULL;
711 if (lexer->rec != NULL)
713 lexer->rec->free(lexer->rec);
714 lexer->rec = NULL;
716 ANTLR3_FREE(lexer);
719 /** Implementation of matchs for the lexer, overrides any
727 matchs(pANTLR3_LEXER lexer, ANTLR3_UCHAR * string)
731 if (lexer->input->istream->_LA(lexer->input->istream, 1) != (*string))
733 if (lexer->rec->state->backtracking > 0)
735 lexer->rec->state->failed = ANTLR3_TRUE;
739 lexer->rec->exConstruct(lexer->rec);
740 lexer->rec->state->failed = ANTLR3_TRUE;
744 lexer->recover(lexer);
750 lexer->input->istream->consume(lexer->input->istream);
755 lexer->rec->state->failed = ANTLR3_FALSE;
762 /** Implementation of matchc for the lexer, overrides any
770 matchc(pANTLR3_LEXER lexer, ANTLR3_UCHAR c)
772 if (lexer->input->istream->_LA(lexer->input->istream, 1) == c)
776 lexer->input->istream->consume(lexer->input->istream);
780 lexer->rec->state->failed = ANTLR3_FALSE;
787 if (lexer->rec->state->backtracking > 0)
789 lexer->rec->state->failed = ANTLR3_TRUE;
793 lexer->rec->exConstruct(lexer->rec);
797 lexer->recover(lexer);
802 /** Implementation of match range for the lexer, overrides any
810 matchRange(pANTLR3_LEXER lexer, ANTLR3_UCHAR low, ANTLR3_UCHAR high)
816 c = lexer->input->istream->_LA(lexer->input->istream, 1);
821 lexer->input->istream->consume(lexer->input->istream);
825 lexer->rec->state->failed = ANTLR3_FALSE;
833 if (lexer->rec->state->backtracking > 0)
835 lexer->rec->state->failed = ANTLR3_TRUE;
839 lexer->rec->exConstruct(lexer->rec);
843 lexer->recover(lexer);
849 matchAny (pANTLR3_LEXER lexer)
851 lexer->input->istream->consume(lexer->input->istream);
855 recover (pANTLR3_LEXER lexer)
857 lexer->input->istream->consume(lexer->input->istream);
861 getLine (pANTLR3_LEXER lexer)
863 return lexer->input->getLine(lexer->input);
867 getCharPositionInLine (pANTLR3_LEXER lexer)
869 return lexer->input->charPositionInLine;
872 static ANTLR3_MARKER getCharIndex (pANTLR3_LEXER lexer)
874 return lexer->input->istream->index(lexer->input->istream);
878 getText (pANTLR3_LEXER lexer)
880 if (lexer->rec->state->text)
882 return lexer->rec->state->text;
885 return lexer->input->substr(
886 lexer->input,
887 lexer->rec->state->tokenStartCharIndex,
888 lexer->getCharIndex(lexer) - lexer->input->charByteSize
