1 //===--- Lexer.cpp - C Language Family Lexer ------------------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file implements the Lexer and Token interfaces. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "clang/Lex/Lexer.h" 15 #include "UnicodeCharSets.h" 16 #include "clang/Basic/CharInfo.h" 17 #include "clang/Basic/SourceManager.h" 18 #include "clang/Lex/CodeCompletionHandler.h" 19 #include "clang/Lex/LexDiagnostic.h" 20 #include "clang/Lex/LiteralSupport.h" 21 #include "clang/Lex/Preprocessor.h" 22 #include "llvm/ADT/STLExtras.h" 23 #include "llvm/ADT/StringExtras.h" 24 #include "llvm/ADT/StringSwitch.h" 25 #include "llvm/Support/Compiler.h" 26 #include "llvm/Support/ConvertUTF.h" 27 #include "llvm/Support/MemoryBuffer.h" 28 #include <cstring> 29 using namespace clang; 30 31 //===----------------------------------------------------------------------===// 32 // Token Class Implementation 33 //===----------------------------------------------------------------------===// 34 35 /// isObjCAtKeyword - Return true if we have an ObjC keyword identifier. 36 bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const { 37 if (IdentifierInfo *II = getIdentifierInfo()) 38 return II->getObjCKeywordID() == objcKey; 39 return false; 40 } 41 42 /// getObjCKeywordID - Return the ObjC keyword kind. 43 tok::ObjCKeywordKind Token::getObjCKeywordID() const { 44 IdentifierInfo *specId = getIdentifierInfo(); 45 return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword; 46 } 47 48 49 //===----------------------------------------------------------------------===// 50 // Lexer Class Implementation 51 //===----------------------------------------------------------------------===// 52 53 void Lexer::anchor() { } 54 55 void Lexer::InitLexer(const char *BufStart, const char *BufPtr, 56 const char *BufEnd) { 57 BufferStart = BufStart; 58 BufferPtr = BufPtr; 59 BufferEnd = BufEnd; 60 61 assert(BufEnd[0] == 0 && 62 "We assume that the input buffer has a null character at the end" 63 " to simplify lexing!"); 64 65 // Check whether we have a BOM in the beginning of the buffer. If yes - act 66 // accordingly. Right now we support only UTF-8 with and without BOM, so, just 67 // skip the UTF-8 BOM if it's present. 68 if (BufferStart == BufferPtr) { 69 // Determine the size of the BOM. 70 StringRef Buf(BufferStart, BufferEnd - BufferStart); 71 size_t BOMLength = llvm::StringSwitch<size_t>(Buf) 72 .StartsWith("\xEF\xBB\xBF", 3) // UTF-8 BOM 73 .Default(0); 74 75 // Skip the BOM. 76 BufferPtr += BOMLength; 77 } 78 79 Is_PragmaLexer = false; 80 CurrentConflictMarkerState = CMK_None; 81 82 // Start of the file is a start of line. 83 IsAtStartOfLine = true; 84 IsAtPhysicalStartOfLine = true; 85 86 HasLeadingSpace = false; 87 HasLeadingEmptyMacro = false; 88 89 // We are not after parsing a #. 90 ParsingPreprocessorDirective = false; 91 92 // We are not after parsing #include. 93 ParsingFilename = false; 94 95 // We are not in raw mode. Raw mode disables diagnostics and interpretation 96 // of tokens (e.g. identifiers, thus disabling macro expansion). It is used 97 // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block 98 // or otherwise skipping over tokens. 99 LexingRawMode = false; 100 101 // Default to not keeping comments. 102 ExtendedTokenMode = 0; 103 } 104 105 /// Lexer constructor - Create a new lexer object for the specified buffer 106 /// with the specified preprocessor managing the lexing process. This lexer 107 /// assumes that the associated file buffer and Preprocessor objects will 108 /// outlive it, so it doesn't take ownership of either of them. 109 Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP) 110 : PreprocessorLexer(&PP, FID), 111 FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)), 112 LangOpts(PP.getLangOpts()) { 113 114 InitLexer(InputFile->getBufferStart(), InputFile->getBufferStart(), 115 InputFile->getBufferEnd()); 116 117 resetExtendedTokenMode(); 118 } 119 120 void Lexer::resetExtendedTokenMode() { 121 assert(PP && "Cannot reset token mode without a preprocessor"); 122 if (LangOpts.TraditionalCPP) 123 SetKeepWhitespaceMode(true); 124 else 125 SetCommentRetentionState(PP->getCommentRetentionState()); 126 } 127 128 /// Lexer constructor - Create a new raw lexer object. This object is only 129 /// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text 130 /// range will outlive it, so it doesn't take ownership of it. 131 Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts, 132 const char *BufStart, const char *BufPtr, const char *BufEnd) 133 : FileLoc(fileloc), LangOpts(langOpts) { 134 135 InitLexer(BufStart, BufPtr, BufEnd); 136 137 // We *are* in raw mode. 138 LexingRawMode = true; 139 } 140 141 /// Lexer constructor - Create a new raw lexer object. This object is only 142 /// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text 143 /// range will outlive it, so it doesn't take ownership of it. 144 Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile, 145 const SourceManager &SM, const LangOptions &langOpts) 146 : Lexer(SM.getLocForStartOfFile(FID), langOpts, FromFile->getBufferStart(), 147 FromFile->getBufferStart(), FromFile->getBufferEnd()) {} 148 149 /// Create_PragmaLexer: Lexer constructor - Create a new lexer object for 150 /// _Pragma expansion. This has a variety of magic semantics that this method 151 /// sets up. It returns a new'd Lexer that must be delete'd when done. 152 /// 153 /// On entrance to this routine, TokStartLoc is a macro location which has a 154 /// spelling loc that indicates the bytes to be lexed for the token and an 155 /// expansion location that indicates where all lexed tokens should be 156 /// "expanded from". 157 /// 158 /// TODO: It would really be nice to make _Pragma just be a wrapper around a 159 /// normal lexer that remaps tokens as they fly by. This would require making 160 /// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer 161 /// interface that could handle this stuff. This would pull GetMappedTokenLoc 162 /// out of the critical path of the lexer! 163 /// 164 Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc, 165 SourceLocation ExpansionLocStart, 166 SourceLocation ExpansionLocEnd, 167 unsigned TokLen, Preprocessor &PP) { 168 SourceManager &SM = PP.getSourceManager(); 169 170 // Create the lexer as if we were going to lex the file normally. 171 FileID SpellingFID = SM.getFileID(SpellingLoc); 172 const llvm::MemoryBuffer *InputFile = SM.getBuffer(SpellingFID); 173 Lexer *L = new Lexer(SpellingFID, InputFile, PP); 174 175 // Now that the lexer is created, change the start/end locations so that we 176 // just lex the subsection of the file that we want. This is lexing from a 177 // scratch buffer. 178 const char *StrData = SM.getCharacterData(SpellingLoc); 179 180 L->BufferPtr = StrData; 181 L->BufferEnd = StrData+TokLen; 182 assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!"); 183 184 // Set the SourceLocation with the remapping information. This ensures that 185 // GetMappedTokenLoc will remap the tokens as they are lexed. 186 L->FileLoc = SM.createExpansionLoc(SM.getLocForStartOfFile(SpellingFID), 187 ExpansionLocStart, 188 ExpansionLocEnd, TokLen); 189 190 // Ensure that the lexer thinks it is inside a directive, so that end \n will 191 // return an EOD token. 192 L->ParsingPreprocessorDirective = true; 193 194 // This lexer really is for _Pragma. 195 L->Is_PragmaLexer = true; 196 return L; 197 } 198 199 200 /// Stringify - Convert the specified string into a C string, with surrounding 201 /// ""'s, and with escaped \ and " characters. 202 std::string Lexer::Stringify(const std::string &Str, bool Charify) { 203 std::string Result = Str; 204 char Quote = Charify ? '\'' : '"'; 205 for (unsigned i = 0, e = Result.size(); i != e; ++i) { 206 if (Result[i] == '\\' || Result[i] == Quote) { 207 Result.insert(Result.begin()+i, '\\'); 208 ++i; ++e; 209 } 210 } 211 return Result; 212 } 213 214 /// Stringify - Convert the specified string into a C string by escaping '\' 215 /// and " characters. This does not add surrounding ""'s to the string. 216 void Lexer::Stringify(SmallVectorImpl<char> &Str) { 217 for (unsigned i = 0, e = Str.size(); i != e; ++i) { 218 if (Str[i] == '\\' || Str[i] == '"') { 219 Str.insert(Str.begin()+i, '\\'); 220 ++i; ++e; 221 } 222 } 223 } 224 225 //===----------------------------------------------------------------------===// 226 // Token Spelling 227 //===----------------------------------------------------------------------===// 228 229 /// \brief Slow case of getSpelling. Extract the characters comprising the 230 /// spelling of this token from the provided input buffer. 231 static size_t getSpellingSlow(const Token &Tok, const char *BufPtr, 232 const LangOptions &LangOpts, char *Spelling) { 233 assert(Tok.needsCleaning() && "getSpellingSlow called on simple token"); 234 235 size_t Length = 0; 236 const char *BufEnd = BufPtr + Tok.getLength(); 237 238 if (Tok.is(tok::string_literal)) { 239 // Munch the encoding-prefix and opening double-quote. 240 while (BufPtr < BufEnd) { 241 unsigned Size; 242 Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); 243 BufPtr += Size; 244 245 if (Spelling[Length - 1] == '"') 246 break; 247 } 248 249 // Raw string literals need special handling; trigraph expansion and line 250 // splicing do not occur within their d-char-sequence nor within their 251 // r-char-sequence. 252 if (Length >= 2 && 253 Spelling[Length - 2] == 'R' && Spelling[Length - 1] == '"') { 254 // Search backwards from the end of the token to find the matching closing 255 // quote. 256 const char *RawEnd = BufEnd; 257 do --RawEnd; while (*RawEnd != '"'); 258 size_t RawLength = RawEnd - BufPtr + 1; 259 260 // Everything between the quotes is included verbatim in the spelling. 261 memcpy(Spelling + Length, BufPtr, RawLength); 262 Length += RawLength; 263 BufPtr += RawLength; 264 265 // The rest of the token is lexed normally. 266 } 267 } 268 269 while (BufPtr < BufEnd) { 270 unsigned Size; 271 Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); 272 BufPtr += Size; 273 } 274 275 assert(Length < Tok.getLength() && 276 "NeedsCleaning flag set on token that didn't need cleaning!"); 277 return Length; 278 } 279 280 /// getSpelling() - Return the 'spelling' of this token. The spelling of a 281 /// token are the characters used to represent the token in the source file 282 /// after trigraph expansion and escaped-newline folding. In particular, this 283 /// wants to get the true, uncanonicalized, spelling of things like digraphs 284 /// UCNs, etc. 285 StringRef Lexer::getSpelling(SourceLocation loc, 286 SmallVectorImpl<char> &buffer, 287 const SourceManager &SM, 288 const LangOptions &options, 289 bool *invalid) { 290 // Break down the source location. 291 std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc); 292 293 // Try to the load the file buffer. 294 bool invalidTemp = false; 295 StringRef file = SM.getBufferData(locInfo.first, &invalidTemp); 296 if (invalidTemp) { 297 if (invalid) *invalid = true; 298 return StringRef(); 299 } 300 301 const char *tokenBegin = file.data() + locInfo.second; 302 303 // Lex from the start of the given location. 304 Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options, 305 file.begin(), tokenBegin, file.end()); 306 Token token; 307 lexer.LexFromRawLexer(token); 308 309 unsigned length = token.getLength(); 310 311 // Common case: no need for cleaning. 312 if (!token.needsCleaning()) 313 return StringRef(tokenBegin, length); 314 315 // Hard case, we need to relex the characters into the string. 316 buffer.resize(length); 317 buffer.resize(getSpellingSlow(token, tokenBegin, options, buffer.data())); 318 return StringRef(buffer.data(), buffer.size()); 319 } 320 321 /// getSpelling() - Return the 'spelling' of this token. The spelling of a 322 /// token are the characters used to represent the token in the source file 323 /// after trigraph expansion and escaped-newline folding. In particular, this 324 /// wants to get the true, uncanonicalized, spelling of things like digraphs 325 /// UCNs, etc. 326 std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr, 327 const LangOptions &LangOpts, bool *Invalid) { 328 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 329 330 bool CharDataInvalid = false; 331 const char *TokStart = SourceMgr.getCharacterData(Tok.getLocation(), 332 &CharDataInvalid); 333 if (Invalid) 334 *Invalid = CharDataInvalid; 335 if (CharDataInvalid) 336 return std::string(); 337 338 // If this token contains nothing interesting, return it directly. 339 if (!Tok.needsCleaning()) 340 return std::string(TokStart, TokStart + Tok.getLength()); 341 342 std::string Result; 343 Result.resize(Tok.getLength()); 344 Result.resize(getSpellingSlow(Tok, TokStart, LangOpts, &*Result.begin())); 345 return Result; 346 } 347 348 /// getSpelling - This method is used to get the spelling of a token into a 349 /// preallocated buffer, instead of as an std::string. The caller is required 350 /// to allocate enough space for the token, which is guaranteed to be at least 351 /// Tok.getLength() bytes long. The actual length of the token is returned. 352 /// 353 /// Note that this method may do two possible things: it may either fill in 354 /// the buffer specified with characters, or it may *change the input pointer* 355 /// to point to a constant buffer with the data already in it (avoiding a 356 /// copy). The caller is not allowed to modify the returned buffer pointer 357 /// if an internal buffer is returned. 358 unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer, 359 const SourceManager &SourceMgr, 360 const LangOptions &LangOpts, bool *Invalid) { 361 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 362 363 const char *TokStart = nullptr; 364 // NOTE: this has to be checked *before* testing for an IdentifierInfo. 365 if (Tok.is(tok::raw_identifier)) 366 TokStart = Tok.getRawIdentifier().data(); 367 else if (!Tok.hasUCN()) { 368 if (const IdentifierInfo *II = Tok.getIdentifierInfo()) { 369 // Just return the string from the identifier table, which is very quick. 370 Buffer = II->getNameStart(); 371 return II->getLength(); 372 } 373 } 374 375 // NOTE: this can be checked even after testing for an IdentifierInfo. 376 if (Tok.isLiteral()) 377 TokStart = Tok.getLiteralData(); 378 379 if (!TokStart) { 380 // Compute the start of the token in the input lexer buffer. 381 bool CharDataInvalid = false; 382 TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid); 383 if (Invalid) 384 *Invalid = CharDataInvalid; 385 if (CharDataInvalid) { 386 Buffer = ""; 387 return 0; 388 } 389 } 390 391 // If this token contains nothing interesting, return it directly. 392 if (!Tok.needsCleaning()) { 393 Buffer = TokStart; 394 return Tok.getLength(); 395 } 396 397 // Otherwise, hard case, relex the characters into the string. 398 return getSpellingSlow(Tok, TokStart, LangOpts, const_cast<char*>(Buffer)); 399 } 400 401 402 /// MeasureTokenLength - Relex the token at the specified location and return 403 /// its length in bytes in the input file. If the token needs cleaning (e.g. 404 /// includes a trigraph or an escaped newline) then this count includes bytes 405 /// that are part of that. 406 unsigned Lexer::MeasureTokenLength(SourceLocation Loc, 407 const SourceManager &SM, 408 const LangOptions &LangOpts) { 409 Token TheTok; 410 if (getRawToken(Loc, TheTok, SM, LangOpts)) 411 return 0; 412 return TheTok.getLength(); 413 } 414 415 /// \brief Relex the token at the specified location. 416 /// \returns true if there was a failure, false on success. 417 bool Lexer::getRawToken(SourceLocation Loc, Token &Result, 418 const SourceManager &SM, 419 const LangOptions &LangOpts, 420 bool IgnoreWhiteSpace) { 421 // TODO: this could be special cased for common tokens like identifiers, ')', 422 // etc to make this faster, if it mattered. Just look at StrData[0] to handle 423 // all obviously single-char tokens. This could use 424 // Lexer::isObviouslySimpleCharacter for example to handle identifiers or 425 // something. 426 427 // If this comes from a macro expansion, we really do want the macro name, not 428 // the token this macro expanded to. 429 Loc = SM.getExpansionLoc(Loc); 430 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 431 bool Invalid = false; 432 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 433 if (Invalid) 434 return true; 435 436 const char *StrData = Buffer.data()+LocInfo.second; 437 438 if (!IgnoreWhiteSpace && isWhitespace(StrData[0])) 439 return true; 440 441 // Create a lexer starting at the beginning of this token. 442 Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, 443 Buffer.begin(), StrData, Buffer.end()); 444 TheLexer.SetCommentRetentionState(true); 445 TheLexer.LexFromRawLexer(Result); 446 return false; 447 } 448 449 static SourceLocation getBeginningOfFileToken(SourceLocation Loc, 450 const SourceManager &SM, 451 const LangOptions &LangOpts) { 452 assert(Loc.isFileID()); 453 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 454 if (LocInfo.first.isInvalid()) 455 return Loc; 456 457 bool Invalid = false; 458 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 459 if (Invalid) 460 return Loc; 461 462 // Back up from the current location until we hit the beginning of a line 463 // (or the buffer). We'll relex from that point. 464 const char *BufStart = Buffer.data(); 465 if (LocInfo.second >= Buffer.size()) 466 return Loc; 467 468 const char *StrData = BufStart+LocInfo.second; 469 if (StrData[0] == '\n' || StrData[0] == '\r') 470 return Loc; 471 472 const char *LexStart = StrData; 473 while (LexStart != BufStart) { 474 if (LexStart[0] == '\n' || LexStart[0] == '\r') { 475 ++LexStart; 476 break; 477 } 478 479 --LexStart; 480 } 481 482 // Create a lexer starting at the beginning of this token. 483 SourceLocation LexerStartLoc = Loc.getLocWithOffset(-LocInfo.second); 484 Lexer TheLexer(LexerStartLoc, LangOpts, BufStart, LexStart, Buffer.end()); 485 TheLexer.SetCommentRetentionState(true); 486 487 // Lex tokens until we find the token that contains the source location. 488 Token TheTok; 489 do { 490 TheLexer.LexFromRawLexer(TheTok); 491 492 if (TheLexer.getBufferLocation() > StrData) { 493 // Lexing this token has taken the lexer past the source location we're 494 // looking for. If the current token encompasses our source location, 495 // return the beginning of that token. 496 if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData) 497 return TheTok.getLocation(); 498 499 // We ended up skipping over the source location entirely, which means 500 // that it points into whitespace. We're done here. 501 break; 502 } 503 } while (TheTok.getKind() != tok::eof); 504 505 // We've passed our source location; just return the original source location. 506 return Loc; 507 } 508 509 SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc, 510 const SourceManager &SM, 511 const LangOptions &LangOpts) { 512 if (Loc.isFileID()) 513 return getBeginningOfFileToken(Loc, SM, LangOpts); 514 515 if (!SM.isMacroArgExpansion(Loc)) 516 return Loc; 517 518 SourceLocation FileLoc = SM.getSpellingLoc(Loc); 519 SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts); 520 std::pair<FileID, unsigned> FileLocInfo = SM.getDecomposedLoc(FileLoc); 521 std::pair<FileID, unsigned> BeginFileLocInfo 522 = SM.getDecomposedLoc(BeginFileLoc); 523 assert(FileLocInfo.first == BeginFileLocInfo.first && 524 FileLocInfo.second >= BeginFileLocInfo.second); 525 return Loc.getLocWithOffset(BeginFileLocInfo.second - FileLocInfo.second); 526 } 527 528 namespace { 529 enum PreambleDirectiveKind { 530 PDK_Skipped, 531 PDK_StartIf, 532 PDK_EndIf, 533 PDK_Unknown 534 }; 535 } 536 537 std::pair<unsigned, bool> Lexer::ComputePreamble(StringRef Buffer, 538 const LangOptions &LangOpts, 539 unsigned MaxLines) { 540 // Create a lexer starting at the beginning of the file. Note that we use a 541 // "fake" file source location at offset 1 so that the lexer will track our 542 // position within the file. 543 const unsigned StartOffset = 1; 544 SourceLocation FileLoc = SourceLocation::getFromRawEncoding(StartOffset); 545 Lexer TheLexer(FileLoc, LangOpts, Buffer.begin(), Buffer.begin(), 546 Buffer.end()); 547 TheLexer.SetCommentRetentionState(true); 548 549 // StartLoc will differ from FileLoc if there is a BOM that was skipped. 550 SourceLocation StartLoc = TheLexer.getSourceLocation(); 551 552 bool InPreprocessorDirective = false; 553 Token TheTok; 554 Token IfStartTok; 555 unsigned IfCount = 0; 556 SourceLocation ActiveCommentLoc; 557 558 unsigned MaxLineOffset = 0; 559 if (MaxLines) { 560 const char *CurPtr = Buffer.begin(); 561 unsigned CurLine = 0; 562 while (CurPtr != Buffer.end()) { 563 char ch = *CurPtr++; 564 if (ch == '\n') { 565 ++CurLine; 566 if (CurLine == MaxLines) 567 break; 568 } 569 } 570 if (CurPtr != Buffer.end()) 571 MaxLineOffset = CurPtr - Buffer.begin(); 572 } 573 574 do { 575 TheLexer.LexFromRawLexer(TheTok); 576 577 if (InPreprocessorDirective) { 578 // If we've hit the end of the file, we're done. 579 if (TheTok.getKind() == tok::eof) { 580 break; 581 } 582 583 // If we haven't hit the end of the preprocessor directive, skip this 584 // token. 585 if (!TheTok.isAtStartOfLine()) 586 continue; 587 588 // We've passed the end of the preprocessor directive, and will look 589 // at this token again below. 590 InPreprocessorDirective = false; 591 } 592 593 // Keep track of the # of lines in the preamble. 594 if (TheTok.isAtStartOfLine()) { 595 unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset; 596 597 // If we were asked to limit the number of lines in the preamble, 598 // and we're about to exceed that limit, we're done. 599 if (MaxLineOffset && TokOffset >= MaxLineOffset) 600 break; 601 } 602 603 // Comments are okay; skip over them. 604 if (TheTok.getKind() == tok::comment) { 605 if (ActiveCommentLoc.isInvalid()) 606 ActiveCommentLoc = TheTok.getLocation(); 607 continue; 608 } 609 610 if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) { 611 // This is the start of a preprocessor directive. 612 Token HashTok = TheTok; 613 InPreprocessorDirective = true; 614 ActiveCommentLoc = SourceLocation(); 615 616 // Figure out which directive this is. Since we're lexing raw tokens, 617 // we don't have an identifier table available. Instead, just look at 618 // the raw identifier to recognize and categorize preprocessor directives. 619 TheLexer.LexFromRawLexer(TheTok); 620 if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) { 621 StringRef Keyword = TheTok.getRawIdentifier(); 622 PreambleDirectiveKind PDK 623 = llvm::StringSwitch<PreambleDirectiveKind>(Keyword) 624 .Case("include", PDK_Skipped) 625 .Case("__include_macros", PDK_Skipped) 626 .Case("define", PDK_Skipped) 627 .Case("undef", PDK_Skipped) 628 .Case("line", PDK_Skipped) 629 .Case("error", PDK_Skipped) 630 .Case("pragma", PDK_Skipped) 631 .Case("import", PDK_Skipped) 632 .Case("include_next", PDK_Skipped) 633 .Case("warning", PDK_Skipped) 634 .Case("ident", PDK_Skipped) 635 .Case("sccs", PDK_Skipped) 636 .Case("assert", PDK_Skipped) 637 .Case("unassert", PDK_Skipped) 638 .Case("if", PDK_StartIf) 639 .Case("ifdef", PDK_StartIf) 640 .Case("ifndef", PDK_StartIf) 641 .Case("elif", PDK_Skipped) 642 .Case("else", PDK_Skipped) 643 .Case("endif", PDK_EndIf) 644 .Default(PDK_Unknown); 645 646 switch (PDK) { 647 case PDK_Skipped: 648 continue; 649 650 case PDK_StartIf: 651 if (IfCount == 0) 652 IfStartTok = HashTok; 653 654 ++IfCount; 655 continue; 656 657 case PDK_EndIf: 658 // Mismatched #endif. The preamble ends here. 659 if (IfCount == 0) 660 break; 661 662 --IfCount; 663 continue; 664 665 case PDK_Unknown: 666 // We don't know what this directive is; stop at the '#'. 667 break; 668 } 669 } 670 671 // We only end up here if we didn't recognize the preprocessor 672 // directive or it was one that can't occur in the preamble at this 673 // point. Roll back the current token to the location of the '#'. 674 InPreprocessorDirective = false; 675 TheTok = HashTok; 676 } 677 678 // We hit a token that we don't recognize as being in the 679 // "preprocessing only" part of the file, so we're no longer in 680 // the preamble. 681 break; 682 } while (true); 683 684 SourceLocation End; 685 if (IfCount) 686 End = IfStartTok.getLocation(); 687 else if (ActiveCommentLoc.isValid()) 688 End = ActiveCommentLoc; // don't truncate a decl comment. 689 else 690 End = TheTok.getLocation(); 691 692 return std::make_pair(End.getRawEncoding() - StartLoc.getRawEncoding(), 693 IfCount? IfStartTok.isAtStartOfLine() 694 : TheTok.isAtStartOfLine()); 695 } 696 697 698 /// AdvanceToTokenCharacter - Given a location that specifies the start of a 699 /// token, return a new location that specifies a character within the token. 700 SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart, 701 unsigned CharNo, 702 const SourceManager &SM, 703 const LangOptions &LangOpts) { 704 // Figure out how many physical characters away the specified expansion 705 // character is. This needs to take into consideration newlines and 706 // trigraphs. 707 bool Invalid = false; 708 const char *TokPtr = SM.getCharacterData(TokStart, &Invalid); 709 710 // If they request the first char of the token, we're trivially done. 711 if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr))) 712 return TokStart; 713 714 unsigned PhysOffset = 0; 715 716 // The usual case is that tokens don't contain anything interesting. Skip 717 // over the uninteresting characters. If a token only consists of simple 718 // chars, this method is extremely fast. 719 while (Lexer::isObviouslySimpleCharacter(*TokPtr)) { 720 if (CharNo == 0) 721 return TokStart.getLocWithOffset(PhysOffset); 722 ++TokPtr, --CharNo, ++PhysOffset; 723 } 724 725 // If we have a character that may be a trigraph or escaped newline, use a 726 // lexer to parse it correctly. 727 for (; CharNo; --CharNo) { 728 unsigned Size; 729 Lexer::getCharAndSizeNoWarn(TokPtr, Size, LangOpts); 730 TokPtr += Size; 731 PhysOffset += Size; 732 } 733 734 // Final detail: if we end up on an escaped newline, we want to return the 735 // location of the actual byte of the token. For example foo\<newline>bar 736 // advanced by 3 should return the location of b, not of \\. One compounding 737 // detail of this is that the escape may be made by a trigraph. 738 if (!Lexer::isObviouslySimpleCharacter(*TokPtr)) 739 PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr; 740 741 return TokStart.getLocWithOffset(PhysOffset); 742 } 743 744 /// \brief Computes the source location just past the end of the 745 /// token at this source location. 746 /// 747 /// This routine can be used to produce a source location that 748 /// points just past the end of the token referenced by \p Loc, and 749 /// is generally used when a diagnostic needs to point just after a 750 /// token where it expected something different that it received. If 751 /// the returned source location would not be meaningful (e.g., if 752 /// it points into a macro), this routine returns an invalid 753 /// source location. 754 /// 755 /// \param Offset an offset from the end of the token, where the source 756 /// location should refer to. The default offset (0) produces a source 757 /// location pointing just past the end of the token; an offset of 1 produces 758 /// a source location pointing to the last character in the token, etc. 759 SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset, 760 const SourceManager &SM, 761 const LangOptions &LangOpts) { 762 if (Loc.isInvalid()) 763 return SourceLocation(); 764 765 if (Loc.isMacroID()) { 766 if (Offset > 0 || !isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) 767 return SourceLocation(); // Points inside the macro expansion. 768 } 769 770 unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts); 771 if (Len > Offset) 772 Len = Len - Offset; 773 else 774 return Loc; 775 776 return Loc.getLocWithOffset(Len); 777 } 778 779 /// \brief Returns true if the given MacroID location points at the first 780 /// token of the macro expansion. 781 bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc, 782 const SourceManager &SM, 783 const LangOptions &LangOpts, 784 SourceLocation *MacroBegin) { 785 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 786 787 SourceLocation expansionLoc; 788 if (!SM.isAtStartOfImmediateMacroExpansion(loc, &expansionLoc)) 789 return false; 790 791 if (expansionLoc.isFileID()) { 792 // No other macro expansions, this is the first. 793 if (MacroBegin) 794 *MacroBegin = expansionLoc; 795 return true; 796 } 797 798 return isAtStartOfMacroExpansion(expansionLoc, SM, LangOpts, MacroBegin); 799 } 800 801 /// \brief Returns true if the given MacroID location points at the last 802 /// token of the macro expansion. 803 bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc, 804 const SourceManager &SM, 805 const LangOptions &LangOpts, 806 SourceLocation *MacroEnd) { 807 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 808 809 SourceLocation spellLoc = SM.getSpellingLoc(loc); 810 unsigned tokLen = MeasureTokenLength(spellLoc, SM, LangOpts); 811 if (tokLen == 0) 812 return false; 813 814 SourceLocation afterLoc = loc.getLocWithOffset(tokLen); 815 SourceLocation expansionLoc; 816 if (!SM.isAtEndOfImmediateMacroExpansion(afterLoc, &expansionLoc)) 817 return false; 818 819 if (expansionLoc.isFileID()) { 820 // No other macro expansions. 821 if (MacroEnd) 822 *MacroEnd = expansionLoc; 823 return true; 824 } 825 826 return isAtEndOfMacroExpansion(expansionLoc, SM, LangOpts, MacroEnd); 827 } 828 829 static CharSourceRange makeRangeFromFileLocs(CharSourceRange Range, 830 const SourceManager &SM, 831 const LangOptions &LangOpts) { 832 SourceLocation Begin = Range.getBegin(); 833 SourceLocation End = Range.getEnd(); 834 assert(Begin.isFileID() && End.isFileID()); 835 if (Range.isTokenRange()) { 836 End = Lexer::getLocForEndOfToken(End, 0, SM,LangOpts); 837 if (End.isInvalid()) 838 return CharSourceRange(); 839 } 840 841 // Break down the source locations. 842 FileID FID; 843 unsigned BeginOffs; 844 std::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin); 845 if (FID.isInvalid()) 846 return CharSourceRange(); 847 848 unsigned EndOffs; 849 if (!SM.isInFileID(End, FID, &EndOffs) || 850 BeginOffs > EndOffs) 851 return CharSourceRange(); 852 853 return CharSourceRange::getCharRange(Begin, End); 854 } 855 856 CharSourceRange Lexer::makeFileCharRange(CharSourceRange Range, 857 const SourceManager &SM, 858 const LangOptions &LangOpts) { 859 SourceLocation Begin = Range.getBegin(); 860 SourceLocation End = Range.getEnd(); 861 if (Begin.isInvalid() || End.isInvalid()) 862 return CharSourceRange(); 863 864 if (Begin.isFileID() && End.isFileID()) 865 return makeRangeFromFileLocs(Range, SM, LangOpts); 866 867 if (Begin.isMacroID() && End.isFileID()) { 868 if (!isAtStartOfMacroExpansion(Begin, SM, LangOpts, &Begin)) 869 return CharSourceRange(); 870 Range.setBegin(Begin); 871 return makeRangeFromFileLocs(Range, SM, LangOpts); 872 } 873 874 if (Begin.isFileID() && End.isMacroID()) { 875 if ((Range.isTokenRange() && !isAtEndOfMacroExpansion(End, SM, LangOpts, 876 &End)) || 877 (Range.isCharRange() && !isAtStartOfMacroExpansion(End, SM, LangOpts, 878 &End))) 879 return CharSourceRange(); 880 Range.setEnd(End); 881 return makeRangeFromFileLocs(Range, SM, LangOpts); 882 } 883 884 assert(Begin.isMacroID() && End.isMacroID()); 885 SourceLocation MacroBegin, MacroEnd; 886 if (isAtStartOfMacroExpansion(Begin, SM, LangOpts, &MacroBegin) && 887 ((Range.isTokenRange() && isAtEndOfMacroExpansion(End, SM, LangOpts, 888 &MacroEnd)) || 889 (Range.isCharRange() && isAtStartOfMacroExpansion(End, SM, LangOpts, 890 &MacroEnd)))) { 891 Range.setBegin(MacroBegin); 892 Range.setEnd(MacroEnd); 893 return makeRangeFromFileLocs(Range, SM, LangOpts); 894 } 895 896 bool Invalid = false; 897 const SrcMgr::SLocEntry &BeginEntry = SM.getSLocEntry(SM.getFileID(Begin), 898 &Invalid); 899 if (Invalid) 900 return CharSourceRange(); 901 902 if (BeginEntry.getExpansion().isMacroArgExpansion()) { 903 const SrcMgr::SLocEntry &EndEntry = SM.getSLocEntry(SM.getFileID(End), 904 &Invalid); 905 if (Invalid) 906 return CharSourceRange(); 907 908 if (EndEntry.getExpansion().isMacroArgExpansion() && 909 BeginEntry.getExpansion().getExpansionLocStart() == 910 EndEntry.getExpansion().getExpansionLocStart()) { 911 Range.setBegin(SM.getImmediateSpellingLoc(Begin)); 912 Range.setEnd(SM.getImmediateSpellingLoc(End)); 913 return makeFileCharRange(Range, SM, LangOpts); 914 } 915 } 916 917 return CharSourceRange(); 918 } 919 920 StringRef Lexer::getSourceText(CharSourceRange Range, 921 const SourceManager &SM, 922 const LangOptions &LangOpts, 923 bool *Invalid) { 924 Range = makeFileCharRange(Range, SM, LangOpts); 925 if (Range.isInvalid()) { 926 if (Invalid) *Invalid = true; 927 return StringRef(); 928 } 929 930 // Break down the source location. 931 std::pair<FileID, unsigned> beginInfo = SM.getDecomposedLoc(Range.getBegin()); 932 if (beginInfo.first.isInvalid()) { 933 if (Invalid) *Invalid = true; 934 return StringRef(); 935 } 936 937 unsigned EndOffs; 938 if (!SM.isInFileID(Range.getEnd(), beginInfo.first, &EndOffs) || 939 beginInfo.second > EndOffs) { 940 if (Invalid) *Invalid = true; 941 return StringRef(); 942 } 943 944 // Try to the load the file buffer. 945 bool invalidTemp = false; 946 StringRef file = SM.getBufferData(beginInfo.first, &invalidTemp); 947 if (invalidTemp) { 948 if (Invalid) *Invalid = true; 949 return StringRef(); 950 } 951 952 if (Invalid) *Invalid = false; 953 return file.substr(beginInfo.second, EndOffs - beginInfo.second); 954 } 955 956 StringRef Lexer::getImmediateMacroName(SourceLocation Loc, 957 const SourceManager &SM, 958 const LangOptions &LangOpts) { 959 assert(Loc.isMacroID() && "Only reasonble to call this on macros"); 960 961 // Find the location of the immediate macro expansion. 962 while (1) { 963 FileID FID = SM.getFileID(Loc); 964 const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID); 965 const SrcMgr::ExpansionInfo &Expansion = E->getExpansion(); 966 Loc = Expansion.getExpansionLocStart(); 967 if (!Expansion.isMacroArgExpansion()) 968 break; 969 970 // For macro arguments we need to check that the argument did not come 971 // from an inner macro, e.g: "MAC1( MAC2(foo) )" 972 973 // Loc points to the argument id of the macro definition, move to the 974 // macro expansion. 975 Loc = SM.getImmediateExpansionRange(Loc).first; 976 SourceLocation SpellLoc = Expansion.getSpellingLoc(); 977 if (SpellLoc.isFileID()) 978 break; // No inner macro. 979 980 // If spelling location resides in the same FileID as macro expansion 981 // location, it means there is no inner macro. 982 FileID MacroFID = SM.getFileID(Loc); 983 if (SM.isInFileID(SpellLoc, MacroFID)) 984 break; 985 986 // Argument came from inner macro. 987 Loc = SpellLoc; 988 } 989 990 // Find the spelling location of the start of the non-argument expansion 991 // range. This is where the macro name was spelled in order to begin 992 // expanding this macro. 993 Loc = SM.getSpellingLoc(Loc); 994 995 // Dig out the buffer where the macro name was spelled and the extents of the 996 // name so that we can render it into the expansion note. 997 std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc); 998 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts); 999 StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first); 1000 return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength); 1001 } 1002 1003 bool Lexer::isIdentifierBodyChar(char c, const LangOptions &LangOpts) { 1004 return isIdentifierBody(c, LangOpts.DollarIdents); 1005 } 1006 1007 1008 //===----------------------------------------------------------------------===// 1009 // Diagnostics forwarding code. 1010 //===----------------------------------------------------------------------===// 1011 1012 /// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the 1013 /// lexer buffer was all expanded at a single point, perform the mapping. 1014 /// This is currently only used for _Pragma implementation, so it is the slow 1015 /// path of the hot getSourceLocation method. Do not allow it to be inlined. 1016 static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc( 1017 Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen); 1018 static SourceLocation GetMappedTokenLoc(Preprocessor &PP, 1019 SourceLocation FileLoc, 1020 unsigned CharNo, unsigned TokLen) { 1021 assert(FileLoc.isMacroID() && "Must be a macro expansion"); 1022 1023 // Otherwise, we're lexing "mapped tokens". This is used for things like 1024 // _Pragma handling. Combine the expansion location of FileLoc with the 1025 // spelling location. 1026 SourceManager &SM = PP.getSourceManager(); 1027 1028 // Create a new SLoc which is expanded from Expansion(FileLoc) but whose 1029 // characters come from spelling(FileLoc)+Offset. 1030 SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc); 1031 SpellingLoc = SpellingLoc.getLocWithOffset(CharNo); 1032 1033 // Figure out the expansion loc range, which is the range covered by the 1034 // original _Pragma(...) sequence. 1035 std::pair<SourceLocation,SourceLocation> II = 1036 SM.getImmediateExpansionRange(FileLoc); 1037 1038 return SM.createExpansionLoc(SpellingLoc, II.first, II.second, TokLen); 1039 } 1040 1041 /// getSourceLocation - Return a source location identifier for the specified 1042 /// offset in the current file. 1043 SourceLocation Lexer::getSourceLocation(const char *Loc, 1044 unsigned TokLen) const { 1045 assert(Loc >= BufferStart && Loc <= BufferEnd && 1046 "Location out of range for this buffer!"); 1047 1048 // In the normal case, we're just lexing from a simple file buffer, return 1049 // the file id from FileLoc with the offset specified. 1050 unsigned CharNo = Loc-BufferStart; 1051 if (FileLoc.isFileID()) 1052 return FileLoc.getLocWithOffset(CharNo); 1053 1054 // Otherwise, this is the _Pragma lexer case, which pretends that all of the 1055 // tokens are lexed from where the _Pragma was defined. 1056 assert(PP && "This doesn't work on raw lexers"); 1057 return GetMappedTokenLoc(*PP, FileLoc, CharNo, TokLen); 1058 } 1059 1060 /// Diag - Forwarding function for diagnostics. This translate a source 1061 /// position in the current buffer into a SourceLocation object for rendering. 1062 DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const { 1063 return PP->Diag(getSourceLocation(Loc), DiagID); 1064 } 1065 1066 //===----------------------------------------------------------------------===// 1067 // Trigraph and Escaped Newline Handling Code. 1068 //===----------------------------------------------------------------------===// 1069 1070 /// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair, 1071 /// return the decoded trigraph letter it corresponds to, or '\0' if nothing. 1072 static char GetTrigraphCharForLetter(char Letter) { 1073 switch (Letter) { 1074 default: return 0; 1075 case '=': return '#'; 1076 case ')': return ']'; 1077 case '(': return '['; 1078 case '!': return '|'; 1079 case '\'': return '^'; 1080 case '>': return '}'; 1081 case '/': return '\\'; 1082 case '<': return '{'; 1083 case '-': return '~'; 1084 } 1085 } 1086 1087 /// DecodeTrigraphChar - If the specified character is a legal trigraph when 1088 /// prefixed with ??, emit a trigraph warning. If trigraphs are enabled, 1089 /// return the result character. Finally, emit a warning about trigraph use 1090 /// whether trigraphs are enabled or not. 1091 static char DecodeTrigraphChar(const char *CP, Lexer *L) { 1092 char Res = GetTrigraphCharForLetter(*CP); 1093 if (!Res || !L) return Res; 1094 1095 if (!L->getLangOpts().Trigraphs) { 1096 if (!L->isLexingRawMode()) 1097 L->Diag(CP-2, diag::trigraph_ignored); 1098 return 0; 1099 } 1100 1101 if (!L->isLexingRawMode()) 1102 L->Diag(CP-2, diag::trigraph_converted) << StringRef(&Res, 1); 1103 return Res; 1104 } 1105 1106 /// getEscapedNewLineSize - Return the size of the specified escaped newline, 1107 /// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a 1108 /// trigraph equivalent on entry to this function. 1109 unsigned Lexer::getEscapedNewLineSize(const char *Ptr) { 1110 unsigned Size = 0; 1111 while (isWhitespace(Ptr[Size])) { 1112 ++Size; 1113 1114 if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r') 1115 continue; 1116 1117 // If this is a \r\n or \n\r, skip the other half. 1118 if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') && 1119 Ptr[Size-1] != Ptr[Size]) 1120 ++Size; 1121 1122 return Size; 1123 } 1124 1125 // Not an escaped newline, must be a \t or something else. 1126 return 0; 1127 } 1128 1129 /// SkipEscapedNewLines - If P points to an escaped newline (or a series of 1130 /// them), skip over them and return the first non-escaped-newline found, 1131 /// otherwise return P. 1132 const char *Lexer::SkipEscapedNewLines(const char *P) { 1133 while (1) { 1134 const char *AfterEscape; 1135 if (*P == '\\') { 1136 AfterEscape = P+1; 1137 } else if (*P == '?') { 1138 // If not a trigraph for escape, bail out. 1139 if (P[1] != '?' || P[2] != '/') 1140 return P; 1141 AfterEscape = P+3; 1142 } else { 1143 return P; 1144 } 1145 1146 unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape); 1147 if (NewLineSize == 0) return P; 1148 P = AfterEscape+NewLineSize; 1149 } 1150 } 1151 1152 /// \brief Checks that the given token is the first token that occurs after the 1153 /// given location (this excludes comments and whitespace). Returns the location 1154 /// immediately after the specified token. If the token is not found or the 1155 /// location is inside a macro, the returned source location will be invalid. 1156 SourceLocation Lexer::findLocationAfterToken(SourceLocation Loc, 1157 tok::TokenKind TKind, 1158 const SourceManager &SM, 1159 const LangOptions &LangOpts, 1160 bool SkipTrailingWhitespaceAndNewLine) { 1161 if (Loc.isMacroID()) { 1162 if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) 1163 return SourceLocation(); 1164 } 1165 Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts); 1166 1167 // Break down the source location. 1168 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 1169 1170 // Try to load the file buffer. 1171 bool InvalidTemp = false; 1172 StringRef File = SM.getBufferData(LocInfo.first, &InvalidTemp); 1173 if (InvalidTemp) 1174 return SourceLocation(); 1175 1176 const char *TokenBegin = File.data() + LocInfo.second; 1177 1178 // Lex from the start of the given location. 1179 Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(), 1180 TokenBegin, File.end()); 1181 // Find the token. 1182 Token Tok; 1183 lexer.LexFromRawLexer(Tok); 1184 if (Tok.isNot(TKind)) 1185 return SourceLocation(); 1186 SourceLocation TokenLoc = Tok.getLocation(); 1187 1188 // Calculate how much whitespace needs to be skipped if any. 1189 unsigned NumWhitespaceChars = 0; 1190 if (SkipTrailingWhitespaceAndNewLine) { 1191 const char *TokenEnd = SM.getCharacterData(TokenLoc) + 1192 Tok.getLength(); 1193 unsigned char C = *TokenEnd; 1194 while (isHorizontalWhitespace(C)) { 1195 C = *(++TokenEnd); 1196 NumWhitespaceChars++; 1197 } 1198 1199 // Skip \r, \n, \r\n, or \n\r 1200 if (C == '\n' || C == '\r') { 1201 char PrevC = C; 1202 C = *(++TokenEnd); 1203 NumWhitespaceChars++; 1204 if ((C == '\n' || C == '\r') && C != PrevC) 1205 NumWhitespaceChars++; 1206 } 1207 } 1208 1209 return TokenLoc.getLocWithOffset(Tok.getLength() + NumWhitespaceChars); 1210 } 1211 1212 /// getCharAndSizeSlow - Peek a single 'character' from the specified buffer, 1213 /// get its size, and return it. This is tricky in several cases: 1214 /// 1. If currently at the start of a trigraph, we warn about the trigraph, 1215 /// then either return the trigraph (skipping 3 chars) or the '?', 1216 /// depending on whether trigraphs are enabled or not. 1217 /// 2. If this is an escaped newline (potentially with whitespace between 1218 /// the backslash and newline), implicitly skip the newline and return 1219 /// the char after it. 1220 /// 1221 /// This handles the slow/uncommon case of the getCharAndSize method. Here we 1222 /// know that we can accumulate into Size, and that we have already incremented 1223 /// Ptr by Size bytes. 1224 /// 1225 /// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should 1226 /// be updated to match. 1227 /// 1228 char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size, 1229 Token *Tok) { 1230 // If we have a slash, look for an escaped newline. 1231 if (Ptr[0] == '\\') { 1232 ++Size; 1233 ++Ptr; 1234 Slash: 1235 // Common case, backslash-char where the char is not whitespace. 1236 if (!isWhitespace(Ptr[0])) return '\\'; 1237 1238 // See if we have optional whitespace characters between the slash and 1239 // newline. 1240 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 1241 // Remember that this token needs to be cleaned. 1242 if (Tok) Tok->setFlag(Token::NeedsCleaning); 1243 1244 // Warn if there was whitespace between the backslash and newline. 1245 if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode()) 1246 Diag(Ptr, diag::backslash_newline_space); 1247 1248 // Found backslash<whitespace><newline>. Parse the char after it. 1249 Size += EscapedNewLineSize; 1250 Ptr += EscapedNewLineSize; 1251 1252 // If the char that we finally got was a \n, then we must have had 1253 // something like \<newline><newline>. We don't want to consume the 1254 // second newline. 1255 if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0') 1256 return ' '; 1257 1258 // Use slow version to accumulate a correct size field. 1259 return getCharAndSizeSlow(Ptr, Size, Tok); 1260 } 1261 1262 // Otherwise, this is not an escaped newline, just return the slash. 1263 return '\\'; 1264 } 1265 1266 // If this is a trigraph, process it. 1267 if (Ptr[0] == '?' && Ptr[1] == '?') { 1268 // If this is actually a legal trigraph (not something like "??x"), emit 1269 // a trigraph warning. If so, and if trigraphs are enabled, return it. 1270 if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : nullptr)) { 1271 // Remember that this token needs to be cleaned. 1272 if (Tok) Tok->setFlag(Token::NeedsCleaning); 1273 1274 Ptr += 3; 1275 Size += 3; 1276 if (C == '\\') goto Slash; 1277 return C; 1278 } 1279 } 1280 1281 // If this is neither, return a single character. 1282 ++Size; 1283 return *Ptr; 1284 } 1285 1286 1287 /// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the 1288 /// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size, 1289 /// and that we have already incremented Ptr by Size bytes. 1290 /// 1291 /// NOTE: When this method is updated, getCharAndSizeSlow (above) should 1292 /// be updated to match. 1293 char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size, 1294 const LangOptions &LangOpts) { 1295 // If we have a slash, look for an escaped newline. 1296 if (Ptr[0] == '\\') { 1297 ++Size; 1298 ++Ptr; 1299 Slash: 1300 // Common case, backslash-char where the char is not whitespace. 1301 if (!isWhitespace(Ptr[0])) return '\\'; 1302 1303 // See if we have optional whitespace characters followed by a newline. 1304 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 1305 // Found backslash<whitespace><newline>. Parse the char after it. 1306 Size += EscapedNewLineSize; 1307 Ptr += EscapedNewLineSize; 1308 1309 // If the char that we finally got was a \n, then we must have had 1310 // something like \<newline><newline>. We don't want to consume the 1311 // second newline. 1312 if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0') 1313 return ' '; 1314 1315 // Use slow version to accumulate a correct size field. 1316 return getCharAndSizeSlowNoWarn(Ptr, Size, LangOpts); 1317 } 1318 1319 // Otherwise, this is not an escaped newline, just return the slash. 1320 return '\\'; 1321 } 1322 1323 // If this is a trigraph, process it. 1324 if (LangOpts.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') { 1325 // If this is actually a legal trigraph (not something like "??x"), return 1326 // it. 1327 if (char C = GetTrigraphCharForLetter(Ptr[2])) { 1328 Ptr += 3; 1329 Size += 3; 1330 if (C == '\\') goto Slash; 1331 return C; 1332 } 1333 } 1334 1335 // If this is neither, return a single character. 1336 ++Size; 1337 return *Ptr; 1338 } 1339 1340 //===----------------------------------------------------------------------===// 1341 // Helper methods for lexing. 1342 //===----------------------------------------------------------------------===// 1343 1344 /// \brief Routine that indiscriminately skips bytes in the source file. 1345 void Lexer::SkipBytes(unsigned Bytes, bool StartOfLine) { 1346 BufferPtr += Bytes; 1347 if (BufferPtr > BufferEnd) 1348 BufferPtr = BufferEnd; 1349 // FIXME: What exactly does the StartOfLine bit mean? There are two 1350 // possible meanings for the "start" of the line: the first token on the 1351 // unexpanded line, or the first token on the expanded line. 1352 IsAtStartOfLine = StartOfLine; 1353 IsAtPhysicalStartOfLine = StartOfLine; 1354 } 1355 1356 static bool isAllowedIDChar(uint32_t C, const LangOptions &LangOpts) { 1357 if (LangOpts.CPlusPlus11 || LangOpts.C11) { 1358 static const llvm::sys::UnicodeCharSet C11AllowedIDChars( 1359 C11AllowedIDCharRanges); 1360 return C11AllowedIDChars.contains(C); 1361 } else if (LangOpts.CPlusPlus) { 1362 static const llvm::sys::UnicodeCharSet CXX03AllowedIDChars( 1363 CXX03AllowedIDCharRanges); 1364 return CXX03AllowedIDChars.contains(C); 1365 } else { 1366 static const llvm::sys::UnicodeCharSet C99AllowedIDChars( 1367 C99AllowedIDCharRanges); 1368 return C99AllowedIDChars.contains(C); 1369 } 1370 } 1371 1372 static bool isAllowedInitiallyIDChar(uint32_t C, const LangOptions &LangOpts) { 1373 assert(isAllowedIDChar(C, LangOpts)); 1374 if (LangOpts.CPlusPlus11 || LangOpts.C11) { 1375 static const llvm::sys::UnicodeCharSet C11DisallowedInitialIDChars( 1376 C11DisallowedInitialIDCharRanges); 1377 return !C11DisallowedInitialIDChars.contains(C); 1378 } else if (LangOpts.CPlusPlus) { 1379 return true; 1380 } else { 1381 static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars( 1382 C99DisallowedInitialIDCharRanges); 1383 return !C99DisallowedInitialIDChars.contains(C); 1384 } 1385 } 1386 1387 static inline CharSourceRange makeCharRange(Lexer &L, const char *Begin, 1388 const char *End) { 1389 return CharSourceRange::getCharRange(L.getSourceLocation(Begin), 1390 L.getSourceLocation(End)); 1391 } 1392 1393 static void maybeDiagnoseIDCharCompat(DiagnosticsEngine &Diags, uint32_t C, 1394 CharSourceRange Range, bool IsFirst) { 1395 // Check C99 compatibility. 1396 if (!Diags.isIgnored(diag::warn_c99_compat_unicode_id, Range.getBegin())) { 1397 enum { 1398 CannotAppearInIdentifier = 0, 1399 CannotStartIdentifier 1400 }; 1401 1402 static const llvm::sys::UnicodeCharSet C99AllowedIDChars( 1403 C99AllowedIDCharRanges); 1404 static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars( 1405 C99DisallowedInitialIDCharRanges); 1406 if (!C99AllowedIDChars.contains(C)) { 1407 Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id) 1408 << Range 1409 << CannotAppearInIdentifier; 1410 } else if (IsFirst && C99DisallowedInitialIDChars.contains(C)) { 1411 Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id) 1412 << Range 1413 << CannotStartIdentifier; 1414 } 1415 } 1416 1417 // Check C++98 compatibility. 1418 if (!Diags.isIgnored(diag::warn_cxx98_compat_unicode_id, Range.getBegin())) { 1419 static const llvm::sys::UnicodeCharSet CXX03AllowedIDChars( 1420 CXX03AllowedIDCharRanges); 1421 if (!CXX03AllowedIDChars.contains(C)) { 1422 Diags.Report(Range.getBegin(), diag::warn_cxx98_compat_unicode_id) 1423 << Range; 1424 } 1425 } 1426 } 1427 1428 bool Lexer::tryConsumeIdentifierUCN(const char *&CurPtr, unsigned Size, 1429 Token &Result) { 1430 const char *UCNPtr = CurPtr + Size; 1431 uint32_t CodePoint = tryReadUCN(UCNPtr, CurPtr, /*Token=*/nullptr); 1432 if (CodePoint == 0 || !isAllowedIDChar(CodePoint, LangOpts)) 1433 return false; 1434 1435 if (!isLexingRawMode()) 1436 maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint, 1437 makeCharRange(*this, CurPtr, UCNPtr), 1438 /*IsFirst=*/false); 1439 1440 Result.setFlag(Token::HasUCN); 1441 if ((UCNPtr - CurPtr == 6 && CurPtr[1] == 'u') || 1442 (UCNPtr - CurPtr == 10 && CurPtr[1] == 'U')) 1443 CurPtr = UCNPtr; 1444 else 1445 while (CurPtr != UCNPtr) 1446 (void)getAndAdvanceChar(CurPtr, Result); 1447 return true; 1448 } 1449 1450 bool Lexer::tryConsumeIdentifierUTF8Char(const char *&CurPtr) { 1451 const char *UnicodePtr = CurPtr; 1452 UTF32 CodePoint; 1453 ConversionResult Result = 1454 llvm::convertUTF8Sequence((const UTF8 **)&UnicodePtr, 1455 (const UTF8 *)BufferEnd, 1456 &CodePoint, 1457 strictConversion); 1458 if (Result != conversionOK || 1459 !isAllowedIDChar(static_cast<uint32_t>(CodePoint), LangOpts)) 1460 return false; 1461 1462 if (!isLexingRawMode()) 1463 maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint, 1464 makeCharRange(*this, CurPtr, UnicodePtr), 1465 /*IsFirst=*/false); 1466 1467 CurPtr = UnicodePtr; 1468 return true; 1469 } 1470 1471 bool Lexer::LexIdentifier(Token &Result, const char *CurPtr) { 1472 // Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$] 1473 unsigned Size; 1474 unsigned char C = *CurPtr++; 1475 while (isIdentifierBody(C)) 1476 C = *CurPtr++; 1477 1478 --CurPtr; // Back up over the skipped character. 1479 1480 // Fast path, no $,\,? in identifier found. '\' might be an escaped newline 1481 // or UCN, and ? might be a trigraph for '\', an escaped newline or UCN. 1482 // 1483 // TODO: Could merge these checks into an InfoTable flag to make the 1484 // comparison cheaper 1485 if (isASCII(C) && C != '\\' && C != '?' && 1486 (C != '$' || !LangOpts.DollarIdents)) { 1487 FinishIdentifier: 1488 const char *IdStart = BufferPtr; 1489 FormTokenWithChars(Result, CurPtr, tok::raw_identifier); 1490 Result.setRawIdentifierData(IdStart); 1491 1492 // If we are in raw mode, return this identifier raw. There is no need to 1493 // look up identifier information or attempt to macro expand it. 1494 if (LexingRawMode) 1495 return true; 1496 1497 // Fill in Result.IdentifierInfo and update the token kind, 1498 // looking up the identifier in the identifier table. 1499 IdentifierInfo *II = PP->LookUpIdentifierInfo(Result); 1500 1501 // Finally, now that we know we have an identifier, pass this off to the 1502 // preprocessor, which may macro expand it or something. 1503 if (II->isHandleIdentifierCase()) 1504 return PP->HandleIdentifier(Result); 1505 1506 return true; 1507 } 1508 1509 // Otherwise, $,\,? in identifier found. Enter slower path. 1510 1511 C = getCharAndSize(CurPtr, Size); 1512 while (1) { 1513 if (C == '$') { 1514 // If we hit a $ and they are not supported in identifiers, we are done. 1515 if (!LangOpts.DollarIdents) goto FinishIdentifier; 1516 1517 // Otherwise, emit a diagnostic and continue. 1518 if (!isLexingRawMode()) 1519 Diag(CurPtr, diag::ext_dollar_in_identifier); 1520 CurPtr = ConsumeChar(CurPtr, Size, Result); 1521 C = getCharAndSize(CurPtr, Size); 1522 continue; 1523 1524 } else if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) { 1525 C = getCharAndSize(CurPtr, Size); 1526 continue; 1527 } else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) { 1528 C = getCharAndSize(CurPtr, Size); 1529 continue; 1530 } else if (!isIdentifierBody(C)) { 1531 goto FinishIdentifier; 1532 } 1533 1534 // Otherwise, this character is good, consume it. 1535 CurPtr = ConsumeChar(CurPtr, Size, Result); 1536 1537 C = getCharAndSize(CurPtr, Size); 1538 while (isIdentifierBody(C)) { 1539 CurPtr = ConsumeChar(CurPtr, Size, Result); 1540 C = getCharAndSize(CurPtr, Size); 1541 } 1542 } 1543 } 1544 1545 /// isHexaLiteral - Return true if Start points to a hex constant. 1546 /// in microsoft mode (where this is supposed to be several different tokens). 1547 bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) { 1548 unsigned Size; 1549 char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, LangOpts); 1550 if (C1 != '0') 1551 return false; 1552 char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, LangOpts); 1553 return (C2 == 'x' || C2 == 'X'); 1554 } 1555 1556 /// LexNumericConstant - Lex the remainder of a integer or floating point 1557 /// constant. From[-1] is the first character lexed. Return the end of the 1558 /// constant. 1559 bool Lexer::LexNumericConstant(Token &Result, const char *CurPtr) { 1560 unsigned Size; 1561 char C = getCharAndSize(CurPtr, Size); 1562 char PrevCh = 0; 1563 while (isPreprocessingNumberBody(C)) { 1564 CurPtr = ConsumeChar(CurPtr, Size, Result); 1565 PrevCh = C; 1566 C = getCharAndSize(CurPtr, Size); 1567 } 1568 1569 // If we fell out, check for a sign, due to 1e+12. If we have one, continue. 1570 if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) { 1571 // If we are in Microsoft mode, don't continue if the constant is hex. 1572 // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1 1573 if (!LangOpts.MicrosoftExt || !isHexaLiteral(BufferPtr, LangOpts)) 1574 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 1575 } 1576 1577 // If we have a hex FP constant, continue. 1578 if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p')) { 1579 // Outside C99, we accept hexadecimal floating point numbers as a 1580 // not-quite-conforming extension. Only do so if this looks like it's 1581 // actually meant to be a hexfloat, and not if it has a ud-suffix. 1582 bool IsHexFloat = true; 1583 if (!LangOpts.C99) { 1584 if (!isHexaLiteral(BufferPtr, LangOpts)) 1585 IsHexFloat = false; 1586 else if (std::find(BufferPtr, CurPtr, '_') != CurPtr) 1587 IsHexFloat = false; 1588 } 1589 if (IsHexFloat) 1590 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 1591 } 1592 1593 // If we have a digit separator, continue. 1594 if (C == '\'' && getLangOpts().CPlusPlus14) { 1595 unsigned NextSize; 1596 char Next = getCharAndSizeNoWarn(CurPtr + Size, NextSize, getLangOpts()); 1597 if (isIdentifierBody(Next)) { 1598 if (!isLexingRawMode()) 1599 Diag(CurPtr, diag::warn_cxx11_compat_digit_separator); 1600 CurPtr = ConsumeChar(CurPtr, Size, Result); 1601 CurPtr = ConsumeChar(CurPtr, NextSize, Result); 1602 return LexNumericConstant(Result, CurPtr); 1603 } 1604 } 1605 1606 // If we have a UCN or UTF-8 character (perhaps in a ud-suffix), continue. 1607 if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) 1608 return LexNumericConstant(Result, CurPtr); 1609 if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) 1610 return LexNumericConstant(Result, CurPtr); 1611 1612 // Update the location of token as well as BufferPtr. 1613 const char *TokStart = BufferPtr; 1614 FormTokenWithChars(Result, CurPtr, tok::numeric_constant); 1615 Result.setLiteralData(TokStart); 1616 return true; 1617 } 1618 1619 /// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes 1620 /// in C++11, or warn on a ud-suffix in C++98. 1621 const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr, 1622 bool IsStringLiteral) { 1623 assert(getLangOpts().CPlusPlus); 1624 1625 // Maximally munch an identifier. 1626 unsigned Size; 1627 char C = getCharAndSize(CurPtr, Size); 1628 bool Consumed = false; 1629 1630 if (!isIdentifierHead(C)) { 1631 if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) 1632 Consumed = true; 1633 else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) 1634 Consumed = true; 1635 else 1636 return CurPtr; 1637 } 1638 1639 if (!getLangOpts().CPlusPlus11) { 1640 if (!isLexingRawMode()) 1641 Diag(CurPtr, 1642 C == '_' ? diag::warn_cxx11_compat_user_defined_literal 1643 : diag::warn_cxx11_compat_reserved_user_defined_literal) 1644 << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); 1645 return CurPtr; 1646 } 1647 1648 // C++11 [lex.ext]p10, [usrlit.suffix]p1: A program containing a ud-suffix 1649 // that does not start with an underscore is ill-formed. As a conforming 1650 // extension, we treat all such suffixes as if they had whitespace before 1651 // them. We assume a suffix beginning with a UCN or UTF-8 character is more 1652 // likely to be a ud-suffix than a macro, however, and accept that. 1653 if (!Consumed) { 1654 bool IsUDSuffix = false; 1655 if (C == '_') 1656 IsUDSuffix = true; 1657 else if (IsStringLiteral && getLangOpts().CPlusPlus14) { 1658 // In C++1y, we need to look ahead a few characters to see if this is a 1659 // valid suffix for a string literal or a numeric literal (this could be 1660 // the 'operator""if' defining a numeric literal operator). 1661 const unsigned MaxStandardSuffixLength = 3; 1662 char Buffer[MaxStandardSuffixLength] = { C }; 1663 unsigned Consumed = Size; 1664 unsigned Chars = 1; 1665 while (true) { 1666 unsigned NextSize; 1667 char Next = getCharAndSizeNoWarn(CurPtr + Consumed, NextSize, 1668 getLangOpts()); 1669 if (!isIdentifierBody(Next)) { 1670 // End of suffix. Check whether this is on the whitelist. 1671 IsUDSuffix = (Chars == 1 && Buffer[0] == 's') || 1672 NumericLiteralParser::isValidUDSuffix( 1673 getLangOpts(), StringRef(Buffer, Chars)); 1674 break; 1675 } 1676 1677 if (Chars == MaxStandardSuffixLength) 1678 // Too long: can't be a standard suffix. 1679 break; 1680 1681 Buffer[Chars++] = Next; 1682 Consumed += NextSize; 1683 } 1684 } 1685 1686 if (!IsUDSuffix) { 1687 if (!isLexingRawMode()) 1688 Diag(CurPtr, getLangOpts().MSVCCompat 1689 ? diag::ext_ms_reserved_user_defined_literal 1690 : diag::ext_reserved_user_defined_literal) 1691 << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); 1692 return CurPtr; 1693 } 1694 1695 CurPtr = ConsumeChar(CurPtr, Size, Result); 1696 } 1697 1698 Result.setFlag(Token::HasUDSuffix); 1699 while (true) { 1700 C = getCharAndSize(CurPtr, Size); 1701 if (isIdentifierBody(C)) { CurPtr = ConsumeChar(CurPtr, Size, Result); } 1702 else if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) {} 1703 else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) {} 1704 else break; 1705 } 1706 1707 return CurPtr; 1708 } 1709 1710 /// LexStringLiteral - Lex the remainder of a string literal, after having lexed 1711 /// either " or L" or u8" or u" or U". 1712 bool Lexer::LexStringLiteral(Token &Result, const char *CurPtr, 1713 tok::TokenKind Kind) { 1714 // Does this string contain the \0 character? 1715 const char *NulCharacter = nullptr; 1716 1717 if (!isLexingRawMode() && 1718 (Kind == tok::utf8_string_literal || 1719 Kind == tok::utf16_string_literal || 1720 Kind == tok::utf32_string_literal)) 1721 Diag(BufferPtr, getLangOpts().CPlusPlus 1722 ? diag::warn_cxx98_compat_unicode_literal 1723 : diag::warn_c99_compat_unicode_literal); 1724 1725 char C = getAndAdvanceChar(CurPtr, Result); 1726 while (C != '"') { 1727 // Skip escaped characters. Escaped newlines will already be processed by 1728 // getAndAdvanceChar. 1729 if (C == '\\') 1730 C = getAndAdvanceChar(CurPtr, Result); 1731 1732 if (C == '\n' || C == '\r' || // Newline. 1733 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 1734 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1735 Diag(BufferPtr, diag::ext_unterminated_string); 1736 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1737 return true; 1738 } 1739 1740 if (C == 0) { 1741 if (isCodeCompletionPoint(CurPtr-1)) { 1742 PP->CodeCompleteNaturalLanguage(); 1743 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1744 cutOffLexing(); 1745 return true; 1746 } 1747 1748 NulCharacter = CurPtr-1; 1749 } 1750 C = getAndAdvanceChar(CurPtr, Result); 1751 } 1752 1753 // If we are in C++11, lex the optional ud-suffix. 1754 if (getLangOpts().CPlusPlus) 1755 CurPtr = LexUDSuffix(Result, CurPtr, true); 1756 1757 // If a nul character existed in the string, warn about it. 1758 if (NulCharacter && !isLexingRawMode()) 1759 Diag(NulCharacter, diag::null_in_string); 1760 1761 // Update the location of the token as well as the BufferPtr instance var. 1762 const char *TokStart = BufferPtr; 1763 FormTokenWithChars(Result, CurPtr, Kind); 1764 Result.setLiteralData(TokStart); 1765 return true; 1766 } 1767 1768 /// LexRawStringLiteral - Lex the remainder of a raw string literal, after 1769 /// having lexed R", LR", u8R", uR", or UR". 1770 bool Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr, 1771 tok::TokenKind Kind) { 1772 // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3: 1773 // Between the initial and final double quote characters of the raw string, 1774 // any transformations performed in phases 1 and 2 (trigraphs, 1775 // universal-character-names, and line splicing) are reverted. 1776 1777 if (!isLexingRawMode()) 1778 Diag(BufferPtr, diag::warn_cxx98_compat_raw_string_literal); 1779 1780 unsigned PrefixLen = 0; 1781 1782 while (PrefixLen != 16 && isRawStringDelimBody(CurPtr[PrefixLen])) 1783 ++PrefixLen; 1784 1785 // If the last character was not a '(', then we didn't lex a valid delimiter. 1786 if (CurPtr[PrefixLen] != '(') { 1787 if (!isLexingRawMode()) { 1788 const char *PrefixEnd = &CurPtr[PrefixLen]; 1789 if (PrefixLen == 16) { 1790 Diag(PrefixEnd, diag::err_raw_delim_too_long); 1791 } else { 1792 Diag(PrefixEnd, diag::err_invalid_char_raw_delim) 1793 << StringRef(PrefixEnd, 1); 1794 } 1795 } 1796 1797 // Search for the next '"' in hopes of salvaging the lexer. Unfortunately, 1798 // it's possible the '"' was intended to be part of the raw string, but 1799 // there's not much we can do about that. 1800 while (1) { 1801 char C = *CurPtr++; 1802 1803 if (C == '"') 1804 break; 1805 if (C == 0 && CurPtr-1 == BufferEnd) { 1806 --CurPtr; 1807 break; 1808 } 1809 } 1810 1811 FormTokenWithChars(Result, CurPtr, tok::unknown); 1812 return true; 1813 } 1814 1815 // Save prefix and move CurPtr past it 1816 const char *Prefix = CurPtr; 1817 CurPtr += PrefixLen + 1; // skip over prefix and '(' 1818 1819 while (1) { 1820 char C = *CurPtr++; 1821 1822 if (C == ')') { 1823 // Check for prefix match and closing quote. 1824 if (strncmp(CurPtr, Prefix, PrefixLen) == 0 && CurPtr[PrefixLen] == '"') { 1825 CurPtr += PrefixLen + 1; // skip over prefix and '"' 1826 break; 1827 } 1828 } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file. 1829 if (!isLexingRawMode()) 1830 Diag(BufferPtr, diag::err_unterminated_raw_string) 1831 << StringRef(Prefix, PrefixLen); 1832 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1833 return true; 1834 } 1835 } 1836 1837 // If we are in C++11, lex the optional ud-suffix. 1838 if (getLangOpts().CPlusPlus) 1839 CurPtr = LexUDSuffix(Result, CurPtr, true); 1840 1841 // Update the location of token as well as BufferPtr. 1842 const char *TokStart = BufferPtr; 1843 FormTokenWithChars(Result, CurPtr, Kind); 1844 Result.setLiteralData(TokStart); 1845 return true; 1846 } 1847 1848 /// LexAngledStringLiteral - Lex the remainder of an angled string literal, 1849 /// after having lexed the '<' character. This is used for #include filenames. 1850 bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) { 1851 // Does this string contain the \0 character? 1852 const char *NulCharacter = nullptr; 1853 const char *AfterLessPos = CurPtr; 1854 char C = getAndAdvanceChar(CurPtr, Result); 1855 while (C != '>') { 1856 // Skip escaped characters. 1857 if (C == '\\') { 1858 // Skip the escaped character. 1859 getAndAdvanceChar(CurPtr, Result); 1860 } else if (C == '\n' || C == '\r' || // Newline. 1861 (C == 0 && (CurPtr-1 == BufferEnd || // End of file. 1862 isCodeCompletionPoint(CurPtr-1)))) { 1863 // If the filename is unterminated, then it must just be a lone < 1864 // character. Return this as such. 1865 FormTokenWithChars(Result, AfterLessPos, tok::less); 1866 return true; 1867 } else if (C == 0) { 1868 NulCharacter = CurPtr-1; 1869 } 1870 C = getAndAdvanceChar(CurPtr, Result); 1871 } 1872 1873 // If a nul character existed in the string, warn about it. 1874 if (NulCharacter && !isLexingRawMode()) 1875 Diag(NulCharacter, diag::null_in_string); 1876 1877 // Update the location of token as well as BufferPtr. 1878 const char *TokStart = BufferPtr; 1879 FormTokenWithChars(Result, CurPtr, tok::angle_string_literal); 1880 Result.setLiteralData(TokStart); 1881 return true; 1882 } 1883 1884 1885 /// LexCharConstant - Lex the remainder of a character constant, after having 1886 /// lexed either ' or L' or u8' or u' or U'. 1887 bool Lexer::LexCharConstant(Token &Result, const char *CurPtr, 1888 tok::TokenKind Kind) { 1889 // Does this character contain the \0 character? 1890 const char *NulCharacter = nullptr; 1891 1892 if (!isLexingRawMode()) { 1893 if (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant) 1894 Diag(BufferPtr, getLangOpts().CPlusPlus 1895 ? diag::warn_cxx98_compat_unicode_literal 1896 : diag::warn_c99_compat_unicode_literal); 1897 else if (Kind == tok::utf8_char_constant) 1898 Diag(BufferPtr, diag::warn_cxx14_compat_u8_character_literal); 1899 } 1900 1901 char C = getAndAdvanceChar(CurPtr, Result); 1902 if (C == '\'') { 1903 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1904 Diag(BufferPtr, diag::ext_empty_character); 1905 FormTokenWithChars(Result, CurPtr, tok::unknown); 1906 return true; 1907 } 1908 1909 while (C != '\'') { 1910 // Skip escaped characters. 1911 if (C == '\\') 1912 C = getAndAdvanceChar(CurPtr, Result); 1913 1914 if (C == '\n' || C == '\r' || // Newline. 1915 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 1916 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1917 Diag(BufferPtr, diag::ext_unterminated_char); 1918 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1919 return true; 1920 } 1921 1922 if (C == 0) { 1923 if (isCodeCompletionPoint(CurPtr-1)) { 1924 PP->CodeCompleteNaturalLanguage(); 1925 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1926 cutOffLexing(); 1927 return true; 1928 } 1929 1930 NulCharacter = CurPtr-1; 1931 } 1932 C = getAndAdvanceChar(CurPtr, Result); 1933 } 1934 1935 // If we are in C++11, lex the optional ud-suffix. 1936 if (getLangOpts().CPlusPlus) 1937 CurPtr = LexUDSuffix(Result, CurPtr, false); 1938 1939 // If a nul character existed in the character, warn about it. 1940 if (NulCharacter && !isLexingRawMode()) 1941 Diag(NulCharacter, diag::null_in_char); 1942 1943 // Update the location of token as well as BufferPtr. 1944 const char *TokStart = BufferPtr; 1945 FormTokenWithChars(Result, CurPtr, Kind); 1946 Result.setLiteralData(TokStart); 1947 return true; 1948 } 1949 1950 /// SkipWhitespace - Efficiently skip over a series of whitespace characters. 1951 /// Update BufferPtr to point to the next non-whitespace character and return. 1952 /// 1953 /// This method forms a token and returns true if KeepWhitespaceMode is enabled. 1954 /// 1955 bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr, 1956 bool &TokAtPhysicalStartOfLine) { 1957 // Whitespace - Skip it, then return the token after the whitespace. 1958 bool SawNewline = isVerticalWhitespace(CurPtr[-1]); 1959 1960 unsigned char Char = *CurPtr; 1961 1962 // Skip consecutive spaces efficiently. 1963 while (1) { 1964 // Skip horizontal whitespace very aggressively. 1965 while (isHorizontalWhitespace(Char)) 1966 Char = *++CurPtr; 1967 1968 // Otherwise if we have something other than whitespace, we're done. 1969 if (!isVerticalWhitespace(Char)) 1970 break; 1971 1972 if (ParsingPreprocessorDirective) { 1973 // End of preprocessor directive line, let LexTokenInternal handle this. 1974 BufferPtr = CurPtr; 1975 return false; 1976 } 1977 1978 // OK, but handle newline. 1979 SawNewline = true; 1980 Char = *++CurPtr; 1981 } 1982 1983 // If the client wants us to return whitespace, return it now. 1984 if (isKeepWhitespaceMode()) { 1985 FormTokenWithChars(Result, CurPtr, tok::unknown); 1986 if (SawNewline) { 1987 IsAtStartOfLine = true; 1988 IsAtPhysicalStartOfLine = true; 1989 } 1990 // FIXME: The next token will not have LeadingSpace set. 1991 return true; 1992 } 1993 1994 // If this isn't immediately after a newline, there is leading space. 1995 char PrevChar = CurPtr[-1]; 1996 bool HasLeadingSpace = !isVerticalWhitespace(PrevChar); 1997 1998 Result.setFlagValue(Token::LeadingSpace, HasLeadingSpace); 1999 if (SawNewline) { 2000 Result.setFlag(Token::StartOfLine); 2001 TokAtPhysicalStartOfLine = true; 2002 } 2003 2004 BufferPtr = CurPtr; 2005 return false; 2006 } 2007 2008 /// We have just read the // characters from input. Skip until we find the 2009 /// newline character thats terminate the comment. Then update BufferPtr and 2010 /// return. 2011 /// 2012 /// If we're in KeepCommentMode or any CommentHandler has inserted 2013 /// some tokens, this will store the first token and return true. 2014 bool Lexer::SkipLineComment(Token &Result, const char *CurPtr, 2015 bool &TokAtPhysicalStartOfLine) { 2016 // If Line comments aren't explicitly enabled for this language, emit an 2017 // extension warning. 2018 if (!LangOpts.LineComment && !isLexingRawMode()) { 2019 Diag(BufferPtr, diag::ext_line_comment); 2020 2021 // Mark them enabled so we only emit one warning for this translation 2022 // unit. 2023 LangOpts.LineComment = true; 2024 } 2025 2026 // Scan over the body of the comment. The common case, when scanning, is that 2027 // the comment contains normal ascii characters with nothing interesting in 2028 // them. As such, optimize for this case with the inner loop. 2029 char C; 2030 do { 2031 C = *CurPtr; 2032 // Skip over characters in the fast loop. 2033 while (C != 0 && // Potentially EOF. 2034 C != '\n' && C != '\r') // Newline or DOS-style newline. 2035 C = *++CurPtr; 2036 2037 const char *NextLine = CurPtr; 2038 if (C != 0) { 2039 // We found a newline, see if it's escaped. 2040 const char *EscapePtr = CurPtr-1; 2041 bool HasSpace = false; 2042 while (isHorizontalWhitespace(*EscapePtr)) { // Skip whitespace. 2043 --EscapePtr; 2044 HasSpace = true; 2045 } 2046 2047 if (*EscapePtr == '\\') // Escaped newline. 2048 CurPtr = EscapePtr; 2049 else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' && 2050 EscapePtr[-2] == '?') // Trigraph-escaped newline. 2051 CurPtr = EscapePtr-2; 2052 else 2053 break; // This is a newline, we're done. 2054 2055 // If there was space between the backslash and newline, warn about it. 2056 if (HasSpace && !isLexingRawMode()) 2057 Diag(EscapePtr, diag::backslash_newline_space); 2058 } 2059 2060 // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to 2061 // properly decode the character. Read it in raw mode to avoid emitting 2062 // diagnostics about things like trigraphs. If we see an escaped newline, 2063 // we'll handle it below. 2064 const char *OldPtr = CurPtr; 2065 bool OldRawMode = isLexingRawMode(); 2066 LexingRawMode = true; 2067 C = getAndAdvanceChar(CurPtr, Result); 2068 LexingRawMode = OldRawMode; 2069 2070 // If we only read only one character, then no special handling is needed. 2071 // We're done and can skip forward to the newline. 2072 if (C != 0 && CurPtr == OldPtr+1) { 2073 CurPtr = NextLine; 2074 break; 2075 } 2076 2077 // If we read multiple characters, and one of those characters was a \r or 2078 // \n, then we had an escaped newline within the comment. Emit diagnostic 2079 // unless the next line is also a // comment. 2080 if (CurPtr != OldPtr+1 && C != '/' && CurPtr[0] != '/') { 2081 for (; OldPtr != CurPtr; ++OldPtr) 2082 if (OldPtr[0] == '\n' || OldPtr[0] == '\r') { 2083 // Okay, we found a // comment that ends in a newline, if the next 2084 // line is also a // comment, but has spaces, don't emit a diagnostic. 2085 if (isWhitespace(C)) { 2086 const char *ForwardPtr = CurPtr; 2087 while (isWhitespace(*ForwardPtr)) // Skip whitespace. 2088 ++ForwardPtr; 2089 if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/') 2090 break; 2091 } 2092 2093 if (!isLexingRawMode()) 2094 Diag(OldPtr-1, diag::ext_multi_line_line_comment); 2095 break; 2096 } 2097 } 2098 2099 if (CurPtr == BufferEnd+1) { 2100 --CurPtr; 2101 break; 2102 } 2103 2104 if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 2105 PP->CodeCompleteNaturalLanguage(); 2106 cutOffLexing(); 2107 return false; 2108 } 2109 2110 } while (C != '\n' && C != '\r'); 2111 2112 // Found but did not consume the newline. Notify comment handlers about the 2113 // comment unless we're in a #if 0 block. 2114 if (PP && !isLexingRawMode() && 2115 PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 2116 getSourceLocation(CurPtr)))) { 2117 BufferPtr = CurPtr; 2118 return true; // A token has to be returned. 2119 } 2120 2121 // If we are returning comments as tokens, return this comment as a token. 2122 if (inKeepCommentMode()) 2123 return SaveLineComment(Result, CurPtr); 2124 2125 // If we are inside a preprocessor directive and we see the end of line, 2126 // return immediately, so that the lexer can return this as an EOD token. 2127 if (ParsingPreprocessorDirective || CurPtr == BufferEnd) { 2128 BufferPtr = CurPtr; 2129 return false; 2130 } 2131 2132 // Otherwise, eat the \n character. We don't care if this is a \n\r or 2133 // \r\n sequence. This is an efficiency hack (because we know the \n can't 2134 // contribute to another token), it isn't needed for correctness. Note that 2135 // this is ok even in KeepWhitespaceMode, because we would have returned the 2136 /// comment above in that mode. 2137 ++CurPtr; 2138 2139 // The next returned token is at the start of the line. 2140 Result.setFlag(Token::StartOfLine); 2141 TokAtPhysicalStartOfLine = true; 2142 // No leading whitespace seen so far. 2143 Result.clearFlag(Token::LeadingSpace); 2144 BufferPtr = CurPtr; 2145 return false; 2146 } 2147 2148 /// If in save-comment mode, package up this Line comment in an appropriate 2149 /// way and return it. 2150 bool Lexer::SaveLineComment(Token &Result, const char *CurPtr) { 2151 // If we're not in a preprocessor directive, just return the // comment 2152 // directly. 2153 FormTokenWithChars(Result, CurPtr, tok::comment); 2154 2155 if (!ParsingPreprocessorDirective || LexingRawMode) 2156 return true; 2157 2158 // If this Line-style comment is in a macro definition, transmogrify it into 2159 // a C-style block comment. 2160 bool Invalid = false; 2161 std::string Spelling = PP->getSpelling(Result, &Invalid); 2162 if (Invalid) 2163 return true; 2164 2165 assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not line comment?"); 2166 Spelling[1] = '*'; // Change prefix to "/*". 2167 Spelling += "*/"; // add suffix. 2168 2169 Result.setKind(tok::comment); 2170 PP->CreateString(Spelling, Result, 2171 Result.getLocation(), Result.getLocation()); 2172 return true; 2173 } 2174 2175 /// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline 2176 /// character (either \\n or \\r) is part of an escaped newline sequence. Issue 2177 /// a diagnostic if so. We know that the newline is inside of a block comment. 2178 static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr, 2179 Lexer *L) { 2180 assert(CurPtr[0] == '\n' || CurPtr[0] == '\r'); 2181 2182 // Back up off the newline. 2183 --CurPtr; 2184 2185 // If this is a two-character newline sequence, skip the other character. 2186 if (CurPtr[0] == '\n' || CurPtr[0] == '\r') { 2187 // \n\n or \r\r -> not escaped newline. 2188 if (CurPtr[0] == CurPtr[1]) 2189 return false; 2190 // \n\r or \r\n -> skip the newline. 2191 --CurPtr; 2192 } 2193 2194 // If we have horizontal whitespace, skip over it. We allow whitespace 2195 // between the slash and newline. 2196 bool HasSpace = false; 2197 while (isHorizontalWhitespace(*CurPtr) || *CurPtr == 0) { 2198 --CurPtr; 2199 HasSpace = true; 2200 } 2201 2202 // If we have a slash, we know this is an escaped newline. 2203 if (*CurPtr == '\\') { 2204 if (CurPtr[-1] != '*') return false; 2205 } else { 2206 // It isn't a slash, is it the ?? / trigraph? 2207 if (CurPtr[0] != '/' || CurPtr[-1] != '?' || CurPtr[-2] != '?' || 2208 CurPtr[-3] != '*') 2209 return false; 2210 2211 // This is the trigraph ending the comment. Emit a stern warning! 2212 CurPtr -= 2; 2213 2214 // If no trigraphs are enabled, warn that we ignored this trigraph and 2215 // ignore this * character. 2216 if (!L->getLangOpts().Trigraphs) { 2217 if (!L->isLexingRawMode()) 2218 L->Diag(CurPtr, diag::trigraph_ignored_block_comment); 2219 return false; 2220 } 2221 if (!L->isLexingRawMode()) 2222 L->Diag(CurPtr, diag::trigraph_ends_block_comment); 2223 } 2224 2225 // Warn about having an escaped newline between the */ characters. 2226 if (!L->isLexingRawMode()) 2227 L->Diag(CurPtr, diag::escaped_newline_block_comment_end); 2228 2229 // If there was space between the backslash and newline, warn about it. 2230 if (HasSpace && !L->isLexingRawMode()) 2231 L->Diag(CurPtr, diag::backslash_newline_space); 2232 2233 return true; 2234 } 2235 2236 #ifdef __SSE2__ 2237 #include <emmintrin.h> 2238 #elif __ALTIVEC__ 2239 #include <altivec.h> 2240 #undef bool 2241 #endif 2242 2243 /// We have just read from input the / and * characters that started a comment. 2244 /// Read until we find the * and / characters that terminate the comment. 2245 /// Note that we don't bother decoding trigraphs or escaped newlines in block 2246 /// comments, because they cannot cause the comment to end. The only thing 2247 /// that can happen is the comment could end with an escaped newline between 2248 /// the terminating * and /. 2249 /// 2250 /// If we're in KeepCommentMode or any CommentHandler has inserted 2251 /// some tokens, this will store the first token and return true. 2252 bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr, 2253 bool &TokAtPhysicalStartOfLine) { 2254 // Scan one character past where we should, looking for a '/' character. Once 2255 // we find it, check to see if it was preceded by a *. This common 2256 // optimization helps people who like to put a lot of * characters in their 2257 // comments. 2258 2259 // The first character we get with newlines and trigraphs skipped to handle 2260 // the degenerate /*/ case below correctly if the * has an escaped newline 2261 // after it. 2262 unsigned CharSize; 2263 unsigned char C = getCharAndSize(CurPtr, CharSize); 2264 CurPtr += CharSize; 2265 if (C == 0 && CurPtr == BufferEnd+1) { 2266 if (!isLexingRawMode()) 2267 Diag(BufferPtr, diag::err_unterminated_block_comment); 2268 --CurPtr; 2269 2270 // KeepWhitespaceMode should return this broken comment as a token. Since 2271 // it isn't a well formed comment, just return it as an 'unknown' token. 2272 if (isKeepWhitespaceMode()) { 2273 FormTokenWithChars(Result, CurPtr, tok::unknown); 2274 return true; 2275 } 2276 2277 BufferPtr = CurPtr; 2278 return false; 2279 } 2280 2281 // Check to see if the first character after the '/*' is another /. If so, 2282 // then this slash does not end the block comment, it is part of it. 2283 if (C == '/') 2284 C = *CurPtr++; 2285 2286 while (1) { 2287 // Skip over all non-interesting characters until we find end of buffer or a 2288 // (probably ending) '/' character. 2289 if (CurPtr + 24 < BufferEnd && 2290 // If there is a code-completion point avoid the fast scan because it 2291 // doesn't check for '\0'. 2292 !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) { 2293 // While not aligned to a 16-byte boundary. 2294 while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0) 2295 C = *CurPtr++; 2296 2297 if (C == '/') goto FoundSlash; 2298 2299 #ifdef __SSE2__ 2300 __m128i Slashes = _mm_set1_epi8('/'); 2301 while (CurPtr+16 <= BufferEnd) { 2302 int cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(*(const __m128i*)CurPtr, 2303 Slashes)); 2304 if (cmp != 0) { 2305 // Adjust the pointer to point directly after the first slash. It's 2306 // not necessary to set C here, it will be overwritten at the end of 2307 // the outer loop. 2308 CurPtr += llvm::countTrailingZeros<unsigned>(cmp) + 1; 2309 goto FoundSlash; 2310 } 2311 CurPtr += 16; 2312 } 2313 #elif __ALTIVEC__ 2314 __vector unsigned char Slashes = { 2315 '/', '/', '/', '/', '/', '/', '/', '/', 2316 '/', '/', '/', '/', '/', '/', '/', '/' 2317 }; 2318 while (CurPtr+16 <= BufferEnd && 2319 !vec_any_eq(*(const vector unsigned char*)CurPtr, Slashes)) 2320 CurPtr += 16; 2321 #else 2322 // Scan for '/' quickly. Many block comments are very large. 2323 while (CurPtr[0] != '/' && 2324 CurPtr[1] != '/' && 2325 CurPtr[2] != '/' && 2326 CurPtr[3] != '/' && 2327 CurPtr+4 < BufferEnd) { 2328 CurPtr += 4; 2329 } 2330 #endif 2331 2332 // It has to be one of the bytes scanned, increment to it and read one. 2333 C = *CurPtr++; 2334 } 2335 2336 // Loop to scan the remainder. 2337 while (C != '/' && C != '\0') 2338 C = *CurPtr++; 2339 2340 if (C == '/') { 2341 FoundSlash: 2342 if (CurPtr[-2] == '*') // We found the final */. We're done! 2343 break; 2344 2345 if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) { 2346 if (isEndOfBlockCommentWithEscapedNewLine(CurPtr-2, this)) { 2347 // We found the final */, though it had an escaped newline between the 2348 // * and /. We're done! 2349 break; 2350 } 2351 } 2352 if (CurPtr[0] == '*' && CurPtr[1] != '/') { 2353 // If this is a /* inside of the comment, emit a warning. Don't do this 2354 // if this is a /*/, which will end the comment. This misses cases with 2355 // embedded escaped newlines, but oh well. 2356 if (!isLexingRawMode()) 2357 Diag(CurPtr-1, diag::warn_nested_block_comment); 2358 } 2359 } else if (C == 0 && CurPtr == BufferEnd+1) { 2360 if (!isLexingRawMode()) 2361 Diag(BufferPtr, diag::err_unterminated_block_comment); 2362 // Note: the user probably forgot a */. We could continue immediately 2363 // after the /*, but this would involve lexing a lot of what really is the 2364 // comment, which surely would confuse the parser. 2365 --CurPtr; 2366 2367 // KeepWhitespaceMode should return this broken comment as a token. Since 2368 // it isn't a well formed comment, just return it as an 'unknown' token. 2369 if (isKeepWhitespaceMode()) { 2370 FormTokenWithChars(Result, CurPtr, tok::unknown); 2371 return true; 2372 } 2373 2374 BufferPtr = CurPtr; 2375 return false; 2376 } else if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 2377 PP->CodeCompleteNaturalLanguage(); 2378 cutOffLexing(); 2379 return false; 2380 } 2381 2382 C = *CurPtr++; 2383 } 2384 2385 // Notify comment handlers about the comment unless we're in a #if 0 block. 2386 if (PP && !isLexingRawMode() && 2387 PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 2388 getSourceLocation(CurPtr)))) { 2389 BufferPtr = CurPtr; 2390 return true; // A token has to be returned. 2391 } 2392 2393 // If we are returning comments as tokens, return this comment as a token. 2394 if (inKeepCommentMode()) { 2395 FormTokenWithChars(Result, CurPtr, tok::comment); 2396 return true; 2397 } 2398 2399 // It is common for the tokens immediately after a /**/ comment to be 2400 // whitespace. Instead of going through the big switch, handle it 2401 // efficiently now. This is safe even in KeepWhitespaceMode because we would 2402 // have already returned above with the comment as a token. 2403 if (isHorizontalWhitespace(*CurPtr)) { 2404 SkipWhitespace(Result, CurPtr+1, TokAtPhysicalStartOfLine); 2405 return false; 2406 } 2407 2408 // Otherwise, just return so that the next character will be lexed as a token. 2409 BufferPtr = CurPtr; 2410 Result.setFlag(Token::LeadingSpace); 2411 return false; 2412 } 2413 2414 //===----------------------------------------------------------------------===// 2415 // Primary Lexing Entry Points 2416 //===----------------------------------------------------------------------===// 2417 2418 /// ReadToEndOfLine - Read the rest of the current preprocessor line as an 2419 /// uninterpreted string. This switches the lexer out of directive mode. 2420 void Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) { 2421 assert(ParsingPreprocessorDirective && ParsingFilename == false && 2422 "Must be in a preprocessing directive!"); 2423 Token Tmp; 2424 2425 // CurPtr - Cache BufferPtr in an automatic variable. 2426 const char *CurPtr = BufferPtr; 2427 while (1) { 2428 char Char = getAndAdvanceChar(CurPtr, Tmp); 2429 switch (Char) { 2430 default: 2431 if (Result) 2432 Result->push_back(Char); 2433 break; 2434 case 0: // Null. 2435 // Found end of file? 2436 if (CurPtr-1 != BufferEnd) { 2437 if (isCodeCompletionPoint(CurPtr-1)) { 2438 PP->CodeCompleteNaturalLanguage(); 2439 cutOffLexing(); 2440 return; 2441 } 2442 2443 // Nope, normal character, continue. 2444 if (Result) 2445 Result->push_back(Char); 2446 break; 2447 } 2448 // FALL THROUGH. 2449 case '\r': 2450 case '\n': 2451 // Okay, we found the end of the line. First, back up past the \0, \r, \n. 2452 assert(CurPtr[-1] == Char && "Trigraphs for newline?"); 2453 BufferPtr = CurPtr-1; 2454 2455 // Next, lex the character, which should handle the EOD transition. 2456 Lex(Tmp); 2457 if (Tmp.is(tok::code_completion)) { 2458 if (PP) 2459 PP->CodeCompleteNaturalLanguage(); 2460 Lex(Tmp); 2461 } 2462 assert(Tmp.is(tok::eod) && "Unexpected token!"); 2463 2464 // Finally, we're done; 2465 return; 2466 } 2467 } 2468 } 2469 2470 /// LexEndOfFile - CurPtr points to the end of this file. Handle this 2471 /// condition, reporting diagnostics and handling other edge cases as required. 2472 /// This returns true if Result contains a token, false if PP.Lex should be 2473 /// called again. 2474 bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) { 2475 // If we hit the end of the file while parsing a preprocessor directive, 2476 // end the preprocessor directive first. The next token returned will 2477 // then be the end of file. 2478 if (ParsingPreprocessorDirective) { 2479 // Done parsing the "line". 2480 ParsingPreprocessorDirective = false; 2481 // Update the location of token as well as BufferPtr. 2482 FormTokenWithChars(Result, CurPtr, tok::eod); 2483 2484 // Restore comment saving mode, in case it was disabled for directive. 2485 if (PP) 2486 resetExtendedTokenMode(); 2487 return true; // Have a token. 2488 } 2489 2490 // If we are in raw mode, return this event as an EOF token. Let the caller 2491 // that put us in raw mode handle the event. 2492 if (isLexingRawMode()) { 2493 Result.startToken(); 2494 BufferPtr = BufferEnd; 2495 FormTokenWithChars(Result, BufferEnd, tok::eof); 2496 return true; 2497 } 2498 2499 // Issue diagnostics for unterminated #if and missing newline. 2500 2501 // If we are in a #if directive, emit an error. 2502 while (!ConditionalStack.empty()) { 2503 if (PP->getCodeCompletionFileLoc() != FileLoc) 2504 PP->Diag(ConditionalStack.back().IfLoc, 2505 diag::err_pp_unterminated_conditional); 2506 ConditionalStack.pop_back(); 2507 } 2508 2509 // C99 5.1.1.2p2: If the file is non-empty and didn't end in a newline, issue 2510 // a pedwarn. 2511 if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r')) { 2512 DiagnosticsEngine &Diags = PP->getDiagnostics(); 2513 SourceLocation EndLoc = getSourceLocation(BufferEnd); 2514 unsigned DiagID; 2515 2516 if (LangOpts.CPlusPlus11) { 2517 // C++11 [lex.phases] 2.2 p2 2518 // Prefer the C++98 pedantic compatibility warning over the generic, 2519 // non-extension, user-requested "missing newline at EOF" warning. 2520 if (!Diags.isIgnored(diag::warn_cxx98_compat_no_newline_eof, EndLoc)) { 2521 DiagID = diag::warn_cxx98_compat_no_newline_eof; 2522 } else { 2523 DiagID = diag::warn_no_newline_eof; 2524 } 2525 } else { 2526 DiagID = diag::ext_no_newline_eof; 2527 } 2528 2529 Diag(BufferEnd, DiagID) 2530 << FixItHint::CreateInsertion(EndLoc, "\n"); 2531 } 2532 2533 BufferPtr = CurPtr; 2534 2535 // Finally, let the preprocessor handle this. 2536 return PP->HandleEndOfFile(Result, isPragmaLexer()); 2537 } 2538 2539 /// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from 2540 /// the specified lexer will return a tok::l_paren token, 0 if it is something 2541 /// else and 2 if there are no more tokens in the buffer controlled by the 2542 /// lexer. 2543 unsigned Lexer::isNextPPTokenLParen() { 2544 assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?"); 2545 2546 // Switch to 'skipping' mode. This will ensure that we can lex a token 2547 // without emitting diagnostics, disables macro expansion, and will cause EOF 2548 // to return an EOF token instead of popping the include stack. 2549 LexingRawMode = true; 2550 2551 // Save state that can be changed while lexing so that we can restore it. 2552 const char *TmpBufferPtr = BufferPtr; 2553 bool inPPDirectiveMode = ParsingPreprocessorDirective; 2554 bool atStartOfLine = IsAtStartOfLine; 2555 bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine; 2556 bool leadingSpace = HasLeadingSpace; 2557 2558 Token Tok; 2559 Lex(Tok); 2560 2561 // Restore state that may have changed. 2562 BufferPtr = TmpBufferPtr; 2563 ParsingPreprocessorDirective = inPPDirectiveMode; 2564 HasLeadingSpace = leadingSpace; 2565 IsAtStartOfLine = atStartOfLine; 2566 IsAtPhysicalStartOfLine = atPhysicalStartOfLine; 2567 2568 // Restore the lexer back to non-skipping mode. 2569 LexingRawMode = false; 2570 2571 if (Tok.is(tok::eof)) 2572 return 2; 2573 return Tok.is(tok::l_paren); 2574 } 2575 2576 /// \brief Find the end of a version control conflict marker. 2577 static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd, 2578 ConflictMarkerKind CMK) { 2579 const char *Terminator = CMK == CMK_Perforce ? "<<<<\n" : ">>>>>>>"; 2580 size_t TermLen = CMK == CMK_Perforce ? 5 : 7; 2581 StringRef RestOfBuffer(CurPtr+TermLen, BufferEnd-CurPtr-TermLen); 2582 size_t Pos = RestOfBuffer.find(Terminator); 2583 while (Pos != StringRef::npos) { 2584 // Must occur at start of line. 2585 if (Pos == 0 || 2586 (RestOfBuffer[Pos - 1] != '\r' && RestOfBuffer[Pos - 1] != '\n')) { 2587 RestOfBuffer = RestOfBuffer.substr(Pos+TermLen); 2588 Pos = RestOfBuffer.find(Terminator); 2589 continue; 2590 } 2591 return RestOfBuffer.data()+Pos; 2592 } 2593 return nullptr; 2594 } 2595 2596 /// IsStartOfConflictMarker - If the specified pointer is the start of a version 2597 /// control conflict marker like '<<<<<<<', recognize it as such, emit an error 2598 /// and recover nicely. This returns true if it is a conflict marker and false 2599 /// if not. 2600 bool Lexer::IsStartOfConflictMarker(const char *CurPtr) { 2601 // Only a conflict marker if it starts at the beginning of a line. 2602 if (CurPtr != BufferStart && 2603 CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 2604 return false; 2605 2606 // Check to see if we have <<<<<<< or >>>>. 2607 if ((BufferEnd-CurPtr < 8 || StringRef(CurPtr, 7) != "<<<<<<<") && 2608 (BufferEnd-CurPtr < 6 || StringRef(CurPtr, 5) != ">>>> ")) 2609 return false; 2610 2611 // If we have a situation where we don't care about conflict markers, ignore 2612 // it. 2613 if (CurrentConflictMarkerState || isLexingRawMode()) 2614 return false; 2615 2616 ConflictMarkerKind Kind = *CurPtr == '<' ? CMK_Normal : CMK_Perforce; 2617 2618 // Check to see if there is an ending marker somewhere in the buffer at the 2619 // start of a line to terminate this conflict marker. 2620 if (FindConflictEnd(CurPtr, BufferEnd, Kind)) { 2621 // We found a match. We are really in a conflict marker. 2622 // Diagnose this, and ignore to the end of line. 2623 Diag(CurPtr, diag::err_conflict_marker); 2624 CurrentConflictMarkerState = Kind; 2625 2626 // Skip ahead to the end of line. We know this exists because the 2627 // end-of-conflict marker starts with \r or \n. 2628 while (*CurPtr != '\r' && *CurPtr != '\n') { 2629 assert(CurPtr != BufferEnd && "Didn't find end of line"); 2630 ++CurPtr; 2631 } 2632 BufferPtr = CurPtr; 2633 return true; 2634 } 2635 2636 // No end of conflict marker found. 2637 return false; 2638 } 2639 2640 2641 /// HandleEndOfConflictMarker - If this is a '====' or '||||' or '>>>>', or if 2642 /// it is '<<<<' and the conflict marker started with a '>>>>' marker, then it 2643 /// is the end of a conflict marker. Handle it by ignoring up until the end of 2644 /// the line. This returns true if it is a conflict marker and false if not. 2645 bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) { 2646 // Only a conflict marker if it starts at the beginning of a line. 2647 if (CurPtr != BufferStart && 2648 CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 2649 return false; 2650 2651 // If we have a situation where we don't care about conflict markers, ignore 2652 // it. 2653 if (!CurrentConflictMarkerState || isLexingRawMode()) 2654 return false; 2655 2656 // Check to see if we have the marker (4 characters in a row). 2657 for (unsigned i = 1; i != 4; ++i) 2658 if (CurPtr[i] != CurPtr[0]) 2659 return false; 2660 2661 // If we do have it, search for the end of the conflict marker. This could 2662 // fail if it got skipped with a '#if 0' or something. Note that CurPtr might 2663 // be the end of conflict marker. 2664 if (const char *End = FindConflictEnd(CurPtr, BufferEnd, 2665 CurrentConflictMarkerState)) { 2666 CurPtr = End; 2667 2668 // Skip ahead to the end of line. 2669 while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n') 2670 ++CurPtr; 2671 2672 BufferPtr = CurPtr; 2673 2674 // No longer in the conflict marker. 2675 CurrentConflictMarkerState = CMK_None; 2676 return true; 2677 } 2678 2679 return false; 2680 } 2681 2682 bool Lexer::isCodeCompletionPoint(const char *CurPtr) const { 2683 if (PP && PP->isCodeCompletionEnabled()) { 2684 SourceLocation Loc = FileLoc.getLocWithOffset(CurPtr-BufferStart); 2685 return Loc == PP->getCodeCompletionLoc(); 2686 } 2687 2688 return false; 2689 } 2690 2691 uint32_t Lexer::tryReadUCN(const char *&StartPtr, const char *SlashLoc, 2692 Token *Result) { 2693 unsigned CharSize; 2694 char Kind = getCharAndSize(StartPtr, CharSize); 2695 2696 unsigned NumHexDigits; 2697 if (Kind == 'u') 2698 NumHexDigits = 4; 2699 else if (Kind == 'U') 2700 NumHexDigits = 8; 2701 else 2702 return 0; 2703 2704 if (!LangOpts.CPlusPlus && !LangOpts.C99) { 2705 if (Result && !isLexingRawMode()) 2706 Diag(SlashLoc, diag::warn_ucn_not_valid_in_c89); 2707 return 0; 2708 } 2709 2710 const char *CurPtr = StartPtr + CharSize; 2711 const char *KindLoc = &CurPtr[-1]; 2712 2713 uint32_t CodePoint = 0; 2714 for (unsigned i = 0; i < NumHexDigits; ++i) { 2715 char C = getCharAndSize(CurPtr, CharSize); 2716 2717 unsigned Value = llvm::hexDigitValue(C); 2718 if (Value == -1U) { 2719 if (Result && !isLexingRawMode()) { 2720 if (i == 0) { 2721 Diag(BufferPtr, diag::warn_ucn_escape_no_digits) 2722 << StringRef(KindLoc, 1); 2723 } else { 2724 Diag(BufferPtr, diag::warn_ucn_escape_incomplete); 2725 2726 // If the user wrote \U1234, suggest a fixit to \u. 2727 if (i == 4 && NumHexDigits == 8) { 2728 CharSourceRange URange = makeCharRange(*this, KindLoc, KindLoc + 1); 2729 Diag(KindLoc, diag::note_ucn_four_not_eight) 2730 << FixItHint::CreateReplacement(URange, "u"); 2731 } 2732 } 2733 } 2734 2735 return 0; 2736 } 2737 2738 CodePoint <<= 4; 2739 CodePoint += Value; 2740 2741 CurPtr += CharSize; 2742 } 2743 2744 if (Result) { 2745 Result->setFlag(Token::HasUCN); 2746 if (CurPtr - StartPtr == (ptrdiff_t)NumHexDigits + 2) 2747 StartPtr = CurPtr; 2748 else 2749 while (StartPtr != CurPtr) 2750 (void)getAndAdvanceChar(StartPtr, *Result); 2751 } else { 2752 StartPtr = CurPtr; 2753 } 2754 2755 // Don't apply C family restrictions to UCNs in assembly mode 2756 if (LangOpts.AsmPreprocessor) 2757 return CodePoint; 2758 2759 // C99 6.4.3p2: A universal character name shall not specify a character whose 2760 // short identifier is less than 00A0 other than 0024 ($), 0040 (@), or 2761 // 0060 (`), nor one in the range D800 through DFFF inclusive.) 2762 // C++11 [lex.charset]p2: If the hexadecimal value for a 2763 // universal-character-name corresponds to a surrogate code point (in the 2764 // range 0xD800-0xDFFF, inclusive), the program is ill-formed. Additionally, 2765 // if the hexadecimal value for a universal-character-name outside the 2766 // c-char-sequence, s-char-sequence, or r-char-sequence of a character or 2767 // string literal corresponds to a control character (in either of the 2768 // ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a character in the 2769 // basic source character set, the program is ill-formed. 2770 if (CodePoint < 0xA0) { 2771 if (CodePoint == 0x24 || CodePoint == 0x40 || CodePoint == 0x60) 2772 return CodePoint; 2773 2774 // We don't use isLexingRawMode() here because we need to warn about bad 2775 // UCNs even when skipping preprocessing tokens in a #if block. 2776 if (Result && PP) { 2777 if (CodePoint < 0x20 || CodePoint >= 0x7F) 2778 Diag(BufferPtr, diag::err_ucn_control_character); 2779 else { 2780 char C = static_cast<char>(CodePoint); 2781 Diag(BufferPtr, diag::err_ucn_escape_basic_scs) << StringRef(&C, 1); 2782 } 2783 } 2784 2785 return 0; 2786 2787 } else if (CodePoint >= 0xD800 && CodePoint <= 0xDFFF) { 2788 // C++03 allows UCNs representing surrogate characters. C99 and C++11 don't. 2789 // We don't use isLexingRawMode() here because we need to diagnose bad 2790 // UCNs even when skipping preprocessing tokens in a #if block. 2791 if (Result && PP) { 2792 if (LangOpts.CPlusPlus && !LangOpts.CPlusPlus11) 2793 Diag(BufferPtr, diag::warn_ucn_escape_surrogate); 2794 else 2795 Diag(BufferPtr, diag::err_ucn_escape_invalid); 2796 } 2797 return 0; 2798 } 2799 2800 return CodePoint; 2801 } 2802 2803 bool Lexer::CheckUnicodeWhitespace(Token &Result, uint32_t C, 2804 const char *CurPtr) { 2805 static const llvm::sys::UnicodeCharSet UnicodeWhitespaceChars( 2806 UnicodeWhitespaceCharRanges); 2807 if (!isLexingRawMode() && !PP->isPreprocessedOutput() && 2808 UnicodeWhitespaceChars.contains(C)) { 2809 Diag(BufferPtr, diag::ext_unicode_whitespace) 2810 << makeCharRange(*this, BufferPtr, CurPtr); 2811 2812 Result.setFlag(Token::LeadingSpace); 2813 return true; 2814 } 2815 return false; 2816 } 2817 2818 bool Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) { 2819 if (isAllowedIDChar(C, LangOpts) && isAllowedInitiallyIDChar(C, LangOpts)) { 2820 if (!isLexingRawMode() && !ParsingPreprocessorDirective && 2821 !PP->isPreprocessedOutput()) { 2822 maybeDiagnoseIDCharCompat(PP->getDiagnostics(), C, 2823 makeCharRange(*this, BufferPtr, CurPtr), 2824 /*IsFirst=*/true); 2825 } 2826 2827 MIOpt.ReadToken(); 2828 return LexIdentifier(Result, CurPtr); 2829 } 2830 2831 if (!isLexingRawMode() && !ParsingPreprocessorDirective && 2832 !PP->isPreprocessedOutput() && 2833 !isASCII(*BufferPtr) && !isAllowedIDChar(C, LangOpts)) { 2834 // Non-ASCII characters tend to creep into source code unintentionally. 2835 // Instead of letting the parser complain about the unknown token, 2836 // just drop the character. 2837 // Note that we can /only/ do this when the non-ASCII character is actually 2838 // spelled as Unicode, not written as a UCN. The standard requires that 2839 // we not throw away any possible preprocessor tokens, but there's a 2840 // loophole in the mapping of Unicode characters to basic character set 2841 // characters that allows us to map these particular characters to, say, 2842 // whitespace. 2843 Diag(BufferPtr, diag::err_non_ascii) 2844 << FixItHint::CreateRemoval(makeCharRange(*this, BufferPtr, CurPtr)); 2845 2846 BufferPtr = CurPtr; 2847 return false; 2848 } 2849 2850 // Otherwise, we have an explicit UCN or a character that's unlikely to show 2851 // up by accident. 2852 MIOpt.ReadToken(); 2853 FormTokenWithChars(Result, CurPtr, tok::unknown); 2854 return true; 2855 } 2856 2857 void Lexer::PropagateLineStartLeadingSpaceInfo(Token &Result) { 2858 IsAtStartOfLine = Result.isAtStartOfLine(); 2859 HasLeadingSpace = Result.hasLeadingSpace(); 2860 HasLeadingEmptyMacro = Result.hasLeadingEmptyMacro(); 2861 // Note that this doesn't affect IsAtPhysicalStartOfLine. 2862 } 2863 2864 bool Lexer::Lex(Token &Result) { 2865 // Start a new token. 2866 Result.startToken(); 2867 2868 // Set up misc whitespace flags for LexTokenInternal. 2869 if (IsAtStartOfLine) { 2870 Result.setFlag(Token::StartOfLine); 2871 IsAtStartOfLine = false; 2872 } 2873 2874 if (HasLeadingSpace) { 2875 Result.setFlag(Token::LeadingSpace); 2876 HasLeadingSpace = false; 2877 } 2878 2879 if (HasLeadingEmptyMacro) { 2880 Result.setFlag(Token::LeadingEmptyMacro); 2881 HasLeadingEmptyMacro = false; 2882 } 2883 2884 bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine; 2885 IsAtPhysicalStartOfLine = false; 2886 bool isRawLex = isLexingRawMode(); 2887 (void) isRawLex; 2888 bool returnedToken = LexTokenInternal(Result, atPhysicalStartOfLine); 2889 // (After the LexTokenInternal call, the lexer might be destroyed.) 2890 assert((returnedToken || !isRawLex) && "Raw lex must succeed"); 2891 return returnedToken; 2892 } 2893 2894 /// LexTokenInternal - This implements a simple C family lexer. It is an 2895 /// extremely performance critical piece of code. This assumes that the buffer 2896 /// has a null character at the end of the file. This returns a preprocessing 2897 /// token, not a normal token, as such, it is an internal interface. It assumes 2898 /// that the Flags of result have been cleared before calling this. 2899 bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) { 2900 LexNextToken: 2901 // New token, can't need cleaning yet. 2902 Result.clearFlag(Token::NeedsCleaning); 2903 Result.setIdentifierInfo(nullptr); 2904 2905 // CurPtr - Cache BufferPtr in an automatic variable. 2906 const char *CurPtr = BufferPtr; 2907 2908 // Small amounts of horizontal whitespace is very common between tokens. 2909 if ((*CurPtr == ' ') || (*CurPtr == '\t')) { 2910 ++CurPtr; 2911 while ((*CurPtr == ' ') || (*CurPtr == '\t')) 2912 ++CurPtr; 2913 2914 // If we are keeping whitespace and other tokens, just return what we just 2915 // skipped. The next lexer invocation will return the token after the 2916 // whitespace. 2917 if (isKeepWhitespaceMode()) { 2918 FormTokenWithChars(Result, CurPtr, tok::unknown); 2919 // FIXME: The next token will not have LeadingSpace set. 2920 return true; 2921 } 2922 2923 BufferPtr = CurPtr; 2924 Result.setFlag(Token::LeadingSpace); 2925 } 2926 2927 unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below. 2928 2929 // Read a character, advancing over it. 2930 char Char = getAndAdvanceChar(CurPtr, Result); 2931 tok::TokenKind Kind; 2932 2933 switch (Char) { 2934 case 0: // Null. 2935 // Found end of file? 2936 if (CurPtr-1 == BufferEnd) 2937 return LexEndOfFile(Result, CurPtr-1); 2938 2939 // Check if we are performing code completion. 2940 if (isCodeCompletionPoint(CurPtr-1)) { 2941 // Return the code-completion token. 2942 Result.startToken(); 2943 FormTokenWithChars(Result, CurPtr, tok::code_completion); 2944 return true; 2945 } 2946 2947 if (!isLexingRawMode()) 2948 Diag(CurPtr-1, diag::null_in_file); 2949 Result.setFlag(Token::LeadingSpace); 2950 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) 2951 return true; // KeepWhitespaceMode 2952 2953 // We know the lexer hasn't changed, so just try again with this lexer. 2954 // (We manually eliminate the tail call to avoid recursion.) 2955 goto LexNextToken; 2956 2957 case 26: // DOS & CP/M EOF: "^Z". 2958 // If we're in Microsoft extensions mode, treat this as end of file. 2959 if (LangOpts.MicrosoftExt) 2960 return LexEndOfFile(Result, CurPtr-1); 2961 2962 // If Microsoft extensions are disabled, this is just random garbage. 2963 Kind = tok::unknown; 2964 break; 2965 2966 case '\n': 2967 case '\r': 2968 // If we are inside a preprocessor directive and we see the end of line, 2969 // we know we are done with the directive, so return an EOD token. 2970 if (ParsingPreprocessorDirective) { 2971 // Done parsing the "line". 2972 ParsingPreprocessorDirective = false; 2973 2974 // Restore comment saving mode, in case it was disabled for directive. 2975 if (PP) 2976 resetExtendedTokenMode(); 2977 2978 // Since we consumed a newline, we are back at the start of a line. 2979 IsAtStartOfLine = true; 2980 IsAtPhysicalStartOfLine = true; 2981 2982 Kind = tok::eod; 2983 break; 2984 } 2985 2986 // No leading whitespace seen so far. 2987 Result.clearFlag(Token::LeadingSpace); 2988 2989 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) 2990 return true; // KeepWhitespaceMode 2991 2992 // We only saw whitespace, so just try again with this lexer. 2993 // (We manually eliminate the tail call to avoid recursion.) 2994 goto LexNextToken; 2995 case ' ': 2996 case '\t': 2997 case '\f': 2998 case '\v': 2999 SkipHorizontalWhitespace: 3000 Result.setFlag(Token::LeadingSpace); 3001 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) 3002 return true; // KeepWhitespaceMode 3003 3004 SkipIgnoredUnits: 3005 CurPtr = BufferPtr; 3006 3007 // If the next token is obviously a // or /* */ comment, skip it efficiently 3008 // too (without going through the big switch stmt). 3009 if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() && 3010 LangOpts.LineComment && 3011 (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP)) { 3012 if (SkipLineComment(Result, CurPtr+2, TokAtPhysicalStartOfLine)) 3013 return true; // There is a token to return. 3014 goto SkipIgnoredUnits; 3015 } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) { 3016 if (SkipBlockComment(Result, CurPtr+2, TokAtPhysicalStartOfLine)) 3017 return true; // There is a token to return. 3018 goto SkipIgnoredUnits; 3019 } else if (isHorizontalWhitespace(*CurPtr)) { 3020 goto SkipHorizontalWhitespace; 3021 } 3022 // We only saw whitespace, so just try again with this lexer. 3023 // (We manually eliminate the tail call to avoid recursion.) 3024 goto LexNextToken; 3025 3026 // C99 6.4.4.1: Integer Constants. 3027 // C99 6.4.4.2: Floating Constants. 3028 case '0': case '1': case '2': case '3': case '4': 3029 case '5': case '6': case '7': case '8': case '9': 3030 // Notify MIOpt that we read a non-whitespace/non-comment token. 3031 MIOpt.ReadToken(); 3032 return LexNumericConstant(Result, CurPtr); 3033 3034 case 'u': // Identifier (uber) or C11/C++11 UTF-8 or UTF-16 string literal 3035 // Notify MIOpt that we read a non-whitespace/non-comment token. 3036 MIOpt.ReadToken(); 3037 3038 if (LangOpts.CPlusPlus11 || LangOpts.C11) { 3039 Char = getCharAndSize(CurPtr, SizeTmp); 3040 3041 // UTF-16 string literal 3042 if (Char == '"') 3043 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3044 tok::utf16_string_literal); 3045 3046 // UTF-16 character constant 3047 if (Char == '\'') 3048 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3049 tok::utf16_char_constant); 3050 3051 // UTF-16 raw string literal 3052 if (Char == 'R' && LangOpts.CPlusPlus11 && 3053 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 3054 return LexRawStringLiteral(Result, 3055 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3056 SizeTmp2, Result), 3057 tok::utf16_string_literal); 3058 3059 if (Char == '8') { 3060 char Char2 = getCharAndSize(CurPtr + SizeTmp, SizeTmp2); 3061 3062 // UTF-8 string literal 3063 if (Char2 == '"') 3064 return LexStringLiteral(Result, 3065 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3066 SizeTmp2, Result), 3067 tok::utf8_string_literal); 3068 if (Char2 == '\'' && LangOpts.CPlusPlus1z) 3069 return LexCharConstant( 3070 Result, ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3071 SizeTmp2, Result), 3072 tok::utf8_char_constant); 3073 3074 if (Char2 == 'R' && LangOpts.CPlusPlus11) { 3075 unsigned SizeTmp3; 3076 char Char3 = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 3077 // UTF-8 raw string literal 3078 if (Char3 == '"') { 3079 return LexRawStringLiteral(Result, 3080 ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3081 SizeTmp2, Result), 3082 SizeTmp3, Result), 3083 tok::utf8_string_literal); 3084 } 3085 } 3086 } 3087 } 3088 3089 // treat u like the start of an identifier. 3090 return LexIdentifier(Result, CurPtr); 3091 3092 case 'U': // Identifier (Uber) or C11/C++11 UTF-32 string literal 3093 // Notify MIOpt that we read a non-whitespace/non-comment token. 3094 MIOpt.ReadToken(); 3095 3096 if (LangOpts.CPlusPlus11 || LangOpts.C11) { 3097 Char = getCharAndSize(CurPtr, SizeTmp); 3098 3099 // UTF-32 string literal 3100 if (Char == '"') 3101 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3102 tok::utf32_string_literal); 3103 3104 // UTF-32 character constant 3105 if (Char == '\'') 3106 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3107 tok::utf32_char_constant); 3108 3109 // UTF-32 raw string literal 3110 if (Char == 'R' && LangOpts.CPlusPlus11 && 3111 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 3112 return LexRawStringLiteral(Result, 3113 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3114 SizeTmp2, Result), 3115 tok::utf32_string_literal); 3116 } 3117 3118 // treat U like the start of an identifier. 3119 return LexIdentifier(Result, CurPtr); 3120 3121 case 'R': // Identifier or C++0x raw string literal 3122 // Notify MIOpt that we read a non-whitespace/non-comment token. 3123 MIOpt.ReadToken(); 3124 3125 if (LangOpts.CPlusPlus11) { 3126 Char = getCharAndSize(CurPtr, SizeTmp); 3127 3128 if (Char == '"') 3129 return LexRawStringLiteral(Result, 3130 ConsumeChar(CurPtr, SizeTmp, Result), 3131 tok::string_literal); 3132 } 3133 3134 // treat R like the start of an identifier. 3135 return LexIdentifier(Result, CurPtr); 3136 3137 case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz"). 3138 // Notify MIOpt that we read a non-whitespace/non-comment token. 3139 MIOpt.ReadToken(); 3140 Char = getCharAndSize(CurPtr, SizeTmp); 3141 3142 // Wide string literal. 3143 if (Char == '"') 3144 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3145 tok::wide_string_literal); 3146 3147 // Wide raw string literal. 3148 if (LangOpts.CPlusPlus11 && Char == 'R' && 3149 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 3150 return LexRawStringLiteral(Result, 3151 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3152 SizeTmp2, Result), 3153 tok::wide_string_literal); 3154 3155 // Wide character constant. 3156 if (Char == '\'') 3157 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3158 tok::wide_char_constant); 3159 // FALL THROUGH, treating L like the start of an identifier. 3160 3161 // C99 6.4.2: Identifiers. 3162 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': 3163 case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N': 3164 case 'O': case 'P': case 'Q': /*'R'*/case 'S': case 'T': /*'U'*/ 3165 case 'V': case 'W': case 'X': case 'Y': case 'Z': 3166 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': 3167 case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': 3168 case 'o': case 'p': case 'q': case 'r': case 's': case 't': /*'u'*/ 3169 case 'v': case 'w': case 'x': case 'y': case 'z': 3170 case '_': 3171 // Notify MIOpt that we read a non-whitespace/non-comment token. 3172 MIOpt.ReadToken(); 3173 return LexIdentifier(Result, CurPtr); 3174 3175 case '$': // $ in identifiers. 3176 if (LangOpts.DollarIdents) { 3177 if (!isLexingRawMode()) 3178 Diag(CurPtr-1, diag::ext_dollar_in_identifier); 3179 // Notify MIOpt that we read a non-whitespace/non-comment token. 3180 MIOpt.ReadToken(); 3181 return LexIdentifier(Result, CurPtr); 3182 } 3183 3184 Kind = tok::unknown; 3185 break; 3186 3187 // C99 6.4.4: Character Constants. 3188 case '\'': 3189 // Notify MIOpt that we read a non-whitespace/non-comment token. 3190 MIOpt.ReadToken(); 3191 return LexCharConstant(Result, CurPtr, tok::char_constant); 3192 3193 // C99 6.4.5: String Literals. 3194 case '"': 3195 // Notify MIOpt that we read a non-whitespace/non-comment token. 3196 MIOpt.ReadToken(); 3197 return LexStringLiteral(Result, CurPtr, tok::string_literal); 3198 3199 // C99 6.4.6: Punctuators. 3200 case '?': 3201 Kind = tok::question; 3202 break; 3203 case '[': 3204 Kind = tok::l_square; 3205 break; 3206 case ']': 3207 Kind = tok::r_square; 3208 break; 3209 case '(': 3210 Kind = tok::l_paren; 3211 break; 3212 case ')': 3213 Kind = tok::r_paren; 3214 break; 3215 case '{': 3216 Kind = tok::l_brace; 3217 break; 3218 case '}': 3219 Kind = tok::r_brace; 3220 break; 3221 case '.': 3222 Char = getCharAndSize(CurPtr, SizeTmp); 3223 if (Char >= '0' && Char <= '9') { 3224 // Notify MIOpt that we read a non-whitespace/non-comment token. 3225 MIOpt.ReadToken(); 3226 3227 return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result)); 3228 } else if (LangOpts.CPlusPlus && Char == '*') { 3229 Kind = tok::periodstar; 3230 CurPtr += SizeTmp; 3231 } else if (Char == '.' && 3232 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.') { 3233 Kind = tok::ellipsis; 3234 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3235 SizeTmp2, Result); 3236 } else { 3237 Kind = tok::period; 3238 } 3239 break; 3240 case '&': 3241 Char = getCharAndSize(CurPtr, SizeTmp); 3242 if (Char == '&') { 3243 Kind = tok::ampamp; 3244 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3245 } else if (Char == '=') { 3246 Kind = tok::ampequal; 3247 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3248 } else { 3249 Kind = tok::amp; 3250 } 3251 break; 3252 case '*': 3253 if (getCharAndSize(CurPtr, SizeTmp) == '=') { 3254 Kind = tok::starequal; 3255 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3256 } else { 3257 Kind = tok::star; 3258 } 3259 break; 3260 case '+': 3261 Char = getCharAndSize(CurPtr, SizeTmp); 3262 if (Char == '+') { 3263 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3264 Kind = tok::plusplus; 3265 } else if (Char == '=') { 3266 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3267 Kind = tok::plusequal; 3268 } else { 3269 Kind = tok::plus; 3270 } 3271 break; 3272 case '-': 3273 Char = getCharAndSize(CurPtr, SizeTmp); 3274 if (Char == '-') { // -- 3275 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3276 Kind = tok::minusminus; 3277 } else if (Char == '>' && LangOpts.CPlusPlus && 3278 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*') { // C++ ->* 3279 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3280 SizeTmp2, Result); 3281 Kind = tok::arrowstar; 3282 } else if (Char == '>') { // -> 3283 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3284 Kind = tok::arrow; 3285 } else if (Char == '=') { // -= 3286 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3287 Kind = tok::minusequal; 3288 } else { 3289 Kind = tok::minus; 3290 } 3291 break; 3292 case '~': 3293 Kind = tok::tilde; 3294 break; 3295 case '!': 3296 if (getCharAndSize(CurPtr, SizeTmp) == '=') { 3297 Kind = tok::exclaimequal; 3298 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3299 } else { 3300 Kind = tok::exclaim; 3301 } 3302 break; 3303 case '/': 3304 // 6.4.9: Comments 3305 Char = getCharAndSize(CurPtr, SizeTmp); 3306 if (Char == '/') { // Line comment. 3307 // Even if Line comments are disabled (e.g. in C89 mode), we generally 3308 // want to lex this as a comment. There is one problem with this though, 3309 // that in one particular corner case, this can change the behavior of the 3310 // resultant program. For example, In "foo //**/ bar", C89 would lex 3311 // this as "foo / bar" and langauges with Line comments would lex it as 3312 // "foo". Check to see if the character after the second slash is a '*'. 3313 // If so, we will lex that as a "/" instead of the start of a comment. 3314 // However, we never do this if we are just preprocessing. 3315 bool TreatAsComment = LangOpts.LineComment && 3316 (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP); 3317 if (!TreatAsComment) 3318 if (!(PP && PP->isPreprocessedOutput())) 3319 TreatAsComment = getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*'; 3320 3321 if (TreatAsComment) { 3322 if (SkipLineComment(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3323 TokAtPhysicalStartOfLine)) 3324 return true; // There is a token to return. 3325 3326 // It is common for the tokens immediately after a // comment to be 3327 // whitespace (indentation for the next line). Instead of going through 3328 // the big switch, handle it efficiently now. 3329 goto SkipIgnoredUnits; 3330 } 3331 } 3332 3333 if (Char == '*') { // /**/ comment. 3334 if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3335 TokAtPhysicalStartOfLine)) 3336 return true; // There is a token to return. 3337 3338 // We only saw whitespace, so just try again with this lexer. 3339 // (We manually eliminate the tail call to avoid recursion.) 3340 goto LexNextToken; 3341 } 3342 3343 if (Char == '=') { 3344 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3345 Kind = tok::slashequal; 3346 } else { 3347 Kind = tok::slash; 3348 } 3349 break; 3350 case '%': 3351 Char = getCharAndSize(CurPtr, SizeTmp); 3352 if (Char == '=') { 3353 Kind = tok::percentequal; 3354 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3355 } else if (LangOpts.Digraphs && Char == '>') { 3356 Kind = tok::r_brace; // '%>' -> '}' 3357 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3358 } else if (LangOpts.Digraphs && Char == ':') { 3359 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3360 Char = getCharAndSize(CurPtr, SizeTmp); 3361 if (Char == '%' && getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':') { 3362 Kind = tok::hashhash; // '%:%:' -> '##' 3363 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3364 SizeTmp2, Result); 3365 } else if (Char == '@' && LangOpts.MicrosoftExt) {// %:@ -> #@ -> Charize 3366 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3367 if (!isLexingRawMode()) 3368 Diag(BufferPtr, diag::ext_charize_microsoft); 3369 Kind = tok::hashat; 3370 } else { // '%:' -> '#' 3371 // We parsed a # character. If this occurs at the start of the line, 3372 // it's actually the start of a preprocessing directive. Callback to 3373 // the preprocessor to handle it. 3374 // TODO: -fpreprocessed mode?? 3375 if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer) 3376 goto HandleDirective; 3377 3378 Kind = tok::hash; 3379 } 3380 } else { 3381 Kind = tok::percent; 3382 } 3383 break; 3384 case '<': 3385 Char = getCharAndSize(CurPtr, SizeTmp); 3386 if (ParsingFilename) { 3387 return LexAngledStringLiteral(Result, CurPtr); 3388 } else if (Char == '<') { 3389 char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 3390 if (After == '=') { 3391 Kind = tok::lesslessequal; 3392 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3393 SizeTmp2, Result); 3394 } else if (After == '<' && IsStartOfConflictMarker(CurPtr-1)) { 3395 // If this is actually a '<<<<<<<' version control conflict marker, 3396 // recognize it as such and recover nicely. 3397 goto LexNextToken; 3398 } else if (After == '<' && HandleEndOfConflictMarker(CurPtr-1)) { 3399 // If this is '<<<<' and we're in a Perforce-style conflict marker, 3400 // ignore it. 3401 goto LexNextToken; 3402 } else if (LangOpts.CUDA && After == '<') { 3403 Kind = tok::lesslessless; 3404 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3405 SizeTmp2, Result); 3406 } else { 3407 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3408 Kind = tok::lessless; 3409 } 3410 } else if (Char == '=') { 3411 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3412 Kind = tok::lessequal; 3413 } else if (LangOpts.Digraphs && Char == ':') { // '<:' -> '[' 3414 if (LangOpts.CPlusPlus11 && 3415 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == ':') { 3416 // C++0x [lex.pptoken]p3: 3417 // Otherwise, if the next three characters are <:: and the subsequent 3418 // character is neither : nor >, the < is treated as a preprocessor 3419 // token by itself and not as the first character of the alternative 3420 // token <:. 3421 unsigned SizeTmp3; 3422 char After = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 3423 if (After != ':' && After != '>') { 3424 Kind = tok::less; 3425 if (!isLexingRawMode()) 3426 Diag(BufferPtr, diag::warn_cxx98_compat_less_colon_colon); 3427 break; 3428 } 3429 } 3430 3431 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3432 Kind = tok::l_square; 3433 } else if (LangOpts.Digraphs && Char == '%') { // '<%' -> '{' 3434 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3435 Kind = tok::l_brace; 3436 } else { 3437 Kind = tok::less; 3438 } 3439 break; 3440 case '>': 3441 Char = getCharAndSize(CurPtr, SizeTmp); 3442 if (Char == '=') { 3443 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3444 Kind = tok::greaterequal; 3445 } else if (Char == '>') { 3446 char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 3447 if (After == '=') { 3448 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3449 SizeTmp2, Result); 3450 Kind = tok::greatergreaterequal; 3451 } else if (After == '>' && IsStartOfConflictMarker(CurPtr-1)) { 3452 // If this is actually a '>>>>' conflict marker, recognize it as such 3453 // and recover nicely. 3454 goto LexNextToken; 3455 } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) { 3456 // If this is '>>>>>>>' and we're in a conflict marker, ignore it. 3457 goto LexNextToken; 3458 } else if (LangOpts.CUDA && After == '>') { 3459 Kind = tok::greatergreatergreater; 3460 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3461 SizeTmp2, Result); 3462 } else { 3463 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3464 Kind = tok::greatergreater; 3465 } 3466 3467 } else { 3468 Kind = tok::greater; 3469 } 3470 break; 3471 case '^': 3472 Char = getCharAndSize(CurPtr, SizeTmp); 3473 if (Char == '=') { 3474 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3475 Kind = tok::caretequal; 3476 } else { 3477 Kind = tok::caret; 3478 } 3479 break; 3480 case '|': 3481 Char = getCharAndSize(CurPtr, SizeTmp); 3482 if (Char == '=') { 3483 Kind = tok::pipeequal; 3484 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3485 } else if (Char == '|') { 3486 // If this is '|||||||' and we're in a conflict marker, ignore it. 3487 if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1)) 3488 goto LexNextToken; 3489 Kind = tok::pipepipe; 3490 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3491 } else { 3492 Kind = tok::pipe; 3493 } 3494 break; 3495 case ':': 3496 Char = getCharAndSize(CurPtr, SizeTmp); 3497 if (LangOpts.Digraphs && Char == '>') { 3498 Kind = tok::r_square; // ':>' -> ']' 3499 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3500 } else if (LangOpts.CPlusPlus && Char == ':') { 3501 Kind = tok::coloncolon; 3502 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3503 } else { 3504 Kind = tok::colon; 3505 } 3506 break; 3507 case ';': 3508 Kind = tok::semi; 3509 break; 3510 case '=': 3511 Char = getCharAndSize(CurPtr, SizeTmp); 3512 if (Char == '=') { 3513 // If this is '====' and we're in a conflict marker, ignore it. 3514 if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1)) 3515 goto LexNextToken; 3516 3517 Kind = tok::equalequal; 3518 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3519 } else { 3520 Kind = tok::equal; 3521 } 3522 break; 3523 case ',': 3524 Kind = tok::comma; 3525 break; 3526 case '#': 3527 Char = getCharAndSize(CurPtr, SizeTmp); 3528 if (Char == '#') { 3529 Kind = tok::hashhash; 3530 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3531 } else if (Char == '@' && LangOpts.MicrosoftExt) { // #@ -> Charize 3532 Kind = tok::hashat; 3533 if (!isLexingRawMode()) 3534 Diag(BufferPtr, diag::ext_charize_microsoft); 3535 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3536 } else { 3537 // We parsed a # character. If this occurs at the start of the line, 3538 // it's actually the start of a preprocessing directive. Callback to 3539 // the preprocessor to handle it. 3540 // TODO: -fpreprocessed mode?? 3541 if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer) 3542 goto HandleDirective; 3543 3544 Kind = tok::hash; 3545 } 3546 break; 3547 3548 case '@': 3549 // Objective C support. 3550 if (CurPtr[-1] == '@' && LangOpts.ObjC1) 3551 Kind = tok::at; 3552 else 3553 Kind = tok::unknown; 3554 break; 3555 3556 // UCNs (C99 6.4.3, C++11 [lex.charset]p2) 3557 case '\\': 3558 if (uint32_t CodePoint = tryReadUCN(CurPtr, BufferPtr, &Result)) { 3559 if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) { 3560 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) 3561 return true; // KeepWhitespaceMode 3562 3563 // We only saw whitespace, so just try again with this lexer. 3564 // (We manually eliminate the tail call to avoid recursion.) 3565 goto LexNextToken; 3566 } 3567 3568 return LexUnicode(Result, CodePoint, CurPtr); 3569 } 3570 3571 Kind = tok::unknown; 3572 break; 3573 3574 default: { 3575 if (isASCII(Char)) { 3576 Kind = tok::unknown; 3577 break; 3578 } 3579 3580 UTF32 CodePoint; 3581 3582 // We can't just reset CurPtr to BufferPtr because BufferPtr may point to 3583 // an escaped newline. 3584 --CurPtr; 3585 ConversionResult Status = 3586 llvm::convertUTF8Sequence((const UTF8 **)&CurPtr, 3587 (const UTF8 *)BufferEnd, 3588 &CodePoint, 3589 strictConversion); 3590 if (Status == conversionOK) { 3591 if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) { 3592 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) 3593 return true; // KeepWhitespaceMode 3594 3595 // We only saw whitespace, so just try again with this lexer. 3596 // (We manually eliminate the tail call to avoid recursion.) 3597 goto LexNextToken; 3598 } 3599 return LexUnicode(Result, CodePoint, CurPtr); 3600 } 3601 3602 if (isLexingRawMode() || ParsingPreprocessorDirective || 3603 PP->isPreprocessedOutput()) { 3604 ++CurPtr; 3605 Kind = tok::unknown; 3606 break; 3607 } 3608 3609 // Non-ASCII characters tend to creep into source code unintentionally. 3610 // Instead of letting the parser complain about the unknown token, 3611 // just diagnose the invalid UTF-8, then drop the character. 3612 Diag(CurPtr, diag::err_invalid_utf8); 3613 3614 BufferPtr = CurPtr+1; 3615 // We're pretending the character didn't exist, so just try again with 3616 // this lexer. 3617 // (We manually eliminate the tail call to avoid recursion.) 3618 goto LexNextToken; 3619 } 3620 } 3621 3622 // Notify MIOpt that we read a non-whitespace/non-comment token. 3623 MIOpt.ReadToken(); 3624 3625 // Update the location of token as well as BufferPtr. 3626 FormTokenWithChars(Result, CurPtr, Kind); 3627 return true; 3628 3629 HandleDirective: 3630 // We parsed a # character and it's the start of a preprocessing directive. 3631 3632 FormTokenWithChars(Result, CurPtr, tok::hash); 3633 PP->HandleDirective(Result); 3634 3635 if (PP->hadModuleLoaderFatalFailure()) { 3636 // With a fatal failure in the module loader, we abort parsing. 3637 assert(Result.is(tok::eof) && "Preprocessor did not set tok:eof"); 3638 return true; 3639 } 3640 3641 // We parsed the directive; lex a token with the new state. 3642 return false; 3643 } 3644