Lines Matching refs:lexer
1 //===--- Lexer.cpp - C Language Family Lexer ------------------------------===//
10 // This file implements the Lexer and Token interfaces.
14 // TODO: GCC Diagnostics emitted by the lexer:
27 #include "clang/Lex/Lexer.h"
62 // Lexer Class Implementation
65 void Lexer::anchor() { }
67 void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
113 /// Lexer constructor - Create a new lexer object for the specified buffer
114 /// with the specified preprocessor managing the lexing process. This lexer
117 Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP)
128 void Lexer::resetExtendedTokenMode() {
136 /// Lexer constructor - Create a new raw lexer object. This object is only
137 /// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text
139 Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts,
149 /// Lexer constructor - Create a new raw lexer object. This object is only
150 /// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text
152 Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile,
163 /// Create_PragmaLexer: Lexer constructor - Create a new lexer object for
165 /// sets up. It returns a new'd Lexer that must be delete'd when done.
173 /// normal lexer that remaps tokens as they fly by. This would require making
174 /// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer
176 /// out of the critical path of the lexer!
178 Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc,
184 // Create the lexer as if we were going to lex the file normally.
187 Lexer *L = new Lexer(SpellingFID, InputFile, PP);
189 // Now that the lexer is created, change the start/end locations so that we
204 // Ensure that the lexer thinks it is inside a directive, so that end \n will
208 // This lexer really is for _Pragma.
216 std::string Lexer::Stringify(const std::string &Str, bool Charify) {
230 void Lexer::Stringify(SmallVectorImpl<char> &Str) {
256 Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts);
285 Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts);
299 StringRef Lexer::getSpelling(SourceLocation loc,
318 Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options,
321 lexer.LexFromRawLexer(token);
340 std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr,
372 unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer,
394 // Compute the start of the token in the input lexer buffer.
420 unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
431 bool Lexer::getRawToken(SourceLocation Loc, Token &Result,
437 // Lexer::isObviouslySimpleCharacter for example to handle identifiers or
454 // Create a lexer starting at the beginning of this token.
455 Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts,
495 // Create a lexer starting at the beginning of this token.
497 Lexer TheLexer(LexerStartLoc, LangOpts, BufStart, LexStart, Buffer.end());
506 // Lexing this token has taken the lexer past the source location we're
522 SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
551 Lexer::ComputePreamble(const llvm::MemoryBuffer *Buffer,
553 // Create a lexer starting at the beginning of the file. Note that we use a
554 // "fake" file source location at offset 1 so that the lexer will track our
558 Lexer TheLexer(FileLoc, LangOpts, Buffer->getBufferStart(),
701 SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart,
712 if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr)))
720 while (Lexer::isObviouslySimpleCharacter(*TokPtr)) {
727 // lexer to parse it correctly.
730 Lexer::getCharAndSizeNoWarn(TokPtr, Size, LangOpts);
739 if (!Lexer::isObviouslySimpleCharacter(*TokPtr))
740 PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr;
760 SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset,
771 unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
782 bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc,
808 bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc,
846 End = Lexer::getLocForEndOfToken(End, 0, SM,LangOpts);
866 CharSourceRange Lexer::makeFileCharRange(CharSourceRange Range,
930 StringRef Lexer::getSourceText(CharSourceRange Range,
966 StringRef Lexer::getImmediateMacroName(SourceLocation Loc,
1008 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
1013 bool Lexer::isIdentifierBodyChar(char c, const LangOptions &LangOpts) {
1023 /// lexer buffer was all expanded at a single point, perform the mapping.
1053 SourceLocation Lexer::getSourceLocation(const char *Loc,
1064 // Otherwise, this is the _Pragma lexer case, which pretends that all of the
1072 DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const {
1101 static char DecodeTrigraphChar(const char *CP, Lexer *L) {
1119 unsigned Lexer::getEscapedNewLineSize(const char *Ptr) {
1142 const char *Lexer::SkipEscapedNewLines(const char *P) {
1156 unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape);
1166 SourceLocation Lexer::findLocationAfterToken(SourceLocation Loc,
1172 if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc))
1175 Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts);
1189 Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(),
1193 lexer.LexFromRawLexer(Tok);
1238 char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size,
1303 char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size,
1355 void Lexer::SkipBytes(unsigned Bytes, bool StartOfLine) {
1381 static inline CharSourceRange makeCharRange(Lexer &L, const char *Begin,
1418 void Lexer::LexIdentifier(Token &Result, const char *CurPtr) {
1531 bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) {
1533 char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, LangOpts);
1536 char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, LangOpts);
1543 void Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
1585 const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr) {
1625 void Lexer::LexStringLiteral(Token &Result, const char *CurPtr,
1680 void Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr,
1707 // Search for the next '"' in hopes of salvaging the lexer. Unfortunately,
1759 void Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
1794 void Lexer::LexCharConstant(Token &Result, const char *CurPtr,
1856 bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) {
1908 bool Lexer::SkipLineComment(Token &Result, const char *CurPtr) {
2012 // return immediately, so that the lexer can return this as an EOD token.
2035 bool Lexer::SaveLineComment(Token &Result, const char *CurPtr) {
2064 Lexer *L) {
2137 bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
2303 /// uninterpreted string. This switches the lexer out of directive mode.
2304 void Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) {
2358 bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
2406 /// the specified lexer will return a tok::l_paren token, 0 if it is something
2408 /// lexer.
2409 unsigned Lexer::isNextPPTokenLParen() {
2429 // Restore the lexer back to non-skipping mode.
2461 bool Lexer::IsStartOfConflictMarker(const char *CurPtr) {
2506 bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) {
2543 bool Lexer::isCodeCompletionPoint(const char *CurPtr) const {
2552 uint32_t Lexer::tryReadUCN(const char *&StartPtr, const char *SlashLoc,
2660 void Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) {
2711 /// LexTokenInternal - This implements a simple C family lexer. It is an
2716 void Lexer::LexTokenInternal(Token &Result) {
2732 // skipped. The next lexer invocation will return the token after the
