Lines Matching defs:Lexer
1 //===--- Lexer.cpp - C Language Family Lexer ------------------------------===//
10 // This file implements the Lexer and Token interfaces.
14 // TODO: GCC Diagnostics emitted by the lexer:
27 #include "clang/Lex/Lexer.h"
59 // Lexer Class Implementation
62 void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
110 /// Lexer constructor - Create a new lexer object for the specified buffer
111 /// with the specified preprocessor managing the lexing process. This lexer
114 Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP)
126 /// Lexer constructor - Create a new raw lexer object. This object is only
127 /// suitable for calls to 'LexRawToken'. This lexer assumes that the text
129 Lexer::Lexer(SourceLocation fileloc, const LangOptions &features,
139 /// Lexer constructor - Create a new raw lexer object. This object is only
140 /// suitable for calls to 'LexRawToken'. This lexer assumes that the text
142 Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile,
153 /// Create_PragmaLexer: Lexer constructor - Create a new lexer object for
155 /// sets up. It returns a new'd Lexer that must be delete'd when done.
163 /// normal lexer that remaps tokens as they fly by. This would require making
164 /// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer
166 /// out of the critical path of the lexer!
168 Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc,
174 // Create the lexer as if we were going to lex the file normally.
177 Lexer *L = new Lexer(SpellingFID, InputFile, PP);
179 // Now that the lexer is created, change the start/end locations so that we
194 // Ensure that the lexer thinks it is inside a directive, so that end \n will
198 // This lexer really is for _Pragma.
206 std::string Lexer::Stringify(const std::string &Str, bool Charify) {
220 void Lexer::Stringify(SmallVectorImpl<char> &Str) {
238 StringRef Lexer::getSpelling(SourceLocation loc,
257 Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options,
260 lexer.LexFromRawLexer(token);
274 buffer.push_back(Lexer::getCharAndSizeNoWarn(ti, charSize, options));
286 std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr,
309 Result.push_back(Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features));
327 unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer,
347 // Compute the start of the token in the input lexer buffer.
369 *OutBuf++ = Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features);
386 unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
392 // Lexer::isObviouslySimpleCharacter for example to handle identifiers or
409 // Create a lexer starting at the beginning of this token.
410 Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts,
451 // Create a lexer starting at the beginning of this token.
453 Lexer TheLexer(LexerStartLoc, LangOpts, BufStart, LexStart, Buffer.end());
462 // Lexing this token has taken the lexer past the source location we're
478 SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
507 Lexer::ComputePreamble(const llvm::MemoryBuffer *Buffer,
509 // Create a lexer starting at the beginning of the file. Note that we use a
510 // "fake" file source location at offset 1 so that the lexer will track our
514 Lexer TheLexer(StartLoc, Features, Buffer->getBufferStart(),
655 SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart,
666 if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr)))
674 while (Lexer::isObviouslySimpleCharacter(*TokPtr)) {
681 // lexer to parse it correctly.
684 Lexer::getCharAndSizeNoWarn(TokPtr, Size, Features);
693 if (!Lexer::isObviouslySimpleCharacter(*TokPtr))
694 PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr;
714 SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset,
728 unsigned Len = Lexer::MeasureTokenLength(Loc, SM, Features);
739 bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc,
760 bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc,
939 /// lexer buffer was all expanded at a single point, perform the mapping.
969 SourceLocation Lexer::getSourceLocation(const char *Loc,
980 // Otherwise, this is the _Pragma lexer case, which pretends that all of the
988 DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const {
1017 static char DecodeTrigraphChar(const char *CP, Lexer *L) {
1035 unsigned Lexer::getEscapedNewLineSize(const char *Ptr) {
1058 const char *Lexer::SkipEscapedNewLines(const char *P) {
1072 unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape);
1082 SourceLocation Lexer::findLocationAfterToken(SourceLocation Loc,
1088 if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts))
1092 Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts);
1106 Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(),
1110 lexer.LexFromRawLexer(Tok);
1149 char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size,
1207 char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size,
1253 void Lexer::SkipBytes(unsigned Bytes, bool StartOfLine) {
1260 void Lexer::LexIdentifier(Token &Result, const char *CurPtr) {
1332 char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, Features);
1335 char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, Features);
1342 void Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
1372 void Lexer::LexStringLiteral(Token &Result, const char *CurPtr,
1421 void Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr,
1448 // Search for the next '"' in hopes of salvaging the lexer. Unfortunately,
1496 void Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
1531 void Lexer::LexCharConstant(Token &Result, const char *CurPtr,
1586 bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) {
1633 bool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) {
1747 // return immediately, so that the lexer can return this as an EOD token.
1770 bool Lexer::SaveBCPLComment(Token &Result, const char *CurPtr) {
1799 Lexer *L) {
1872 bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
2031 /// uninterpreted string. This switches the lexer out of directive mode.
2032 std::string Lexer::ReadToEndOfLine() {
2085 bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
2132 /// the specified lexer will return a tok::l_paren token, 0 if it is something
2134 /// lexer.
2135 unsigned Lexer::isNextPPTokenLParen() {
2155 // Restore the lexer back to non-skipping mode.
2187 bool Lexer::IsStartOfConflictMarker(const char *CurPtr) {
2232 bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) {
2269 bool Lexer::isCodeCompletionPoint(const char *CurPtr) const {
2279 /// LexTokenInternal - This implements a simple C family lexer. It is an
2284 void Lexer::LexTokenInternal(Token &Result) {
2300 // skipped. The next lexer invocation will return the token after the
