Home | History | Annotate | Download | only in Lex 
      1 //===--- Preprocess.cpp - C Language Family Preprocessor Implementation ---===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 //  This file implements the Preprocessor interface.
     11 //
     12 //===----------------------------------------------------------------------===//
     13 //
     14 // Options to support:
     15 //   -H       - Print the name of each header file used.
     16 //   -d[DNI] - Dump various things.
     17 //   -fworking-directory - #line's with preprocessor's working dir.
     18 //   -fpreprocessed
     19 //   -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
     20 //   -W*
     21 //   -w
     22 //
     23 // Messages to emit:
     24 //   "Multiple include guards may be useful for:\n"
     25 //
     26 //===----------------------------------------------------------------------===//
     27 
     28 #include "clang/Lex/Preprocessor.h"
     29 #include "MacroArgs.h"
     30 #include "clang/Lex/ExternalPreprocessorSource.h"
     31 #include "clang/Lex/HeaderSearch.h"
     32 #include "clang/Lex/MacroInfo.h"
     33 #include "clang/Lex/Pragma.h"
     34 #include "clang/Lex/PreprocessingRecord.h"
     35 #include "clang/Lex/ScratchBuffer.h"
     36 #include "clang/Lex/LexDiagnostic.h"
     37 #include "clang/Lex/CodeCompletionHandler.h"
     38 #include "clang/Lex/ModuleLoader.h"
     39 #include "clang/Basic/SourceManager.h"
     40 #include "clang/Basic/FileManager.h"
     41 #include "clang/Basic/TargetInfo.h"
     42 #include "llvm/ADT/APFloat.h"
     43 #include "llvm/ADT/SmallString.h"
     44 #include "llvm/Support/MemoryBuffer.h"
     45 #include "llvm/Support/raw_ostream.h"
     46 #include "llvm/Support/Capacity.h"
     47 using namespace clang;
     48 
     49 //===----------------------------------------------------------------------===//
     50 ExternalPreprocessorSource::~ExternalPreprocessorSource() { }
     51 
     52 Preprocessor::Preprocessor(DiagnosticsEngine &diags, LangOptions &opts,
     53                            const TargetInfo *target, SourceManager &SM,
     54                            HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
     55                            IdentifierInfoLookup* IILookup,
     56                            bool OwnsHeaders,
     57                            bool DelayInitialization,
     58                            bool IncrProcessing)
     59   : Diags(&diags), LangOpts(opts), Target(target),FileMgr(Headers.getFileMgr()),
     60     SourceMgr(SM), HeaderInfo(Headers), TheModuleLoader(TheModuleLoader),
     61     ExternalSource(0), Identifiers(opts, IILookup),
     62     IncrementalProcessing(IncrProcessing), CodeComplete(0),
     63     CodeCompletionFile(0), CodeCompletionOffset(0), CodeCompletionReached(0),
     64     SkipMainFilePreamble(0, true), CurPPLexer(0),
     65     CurDirLookup(0), CurLexerKind(CLK_Lexer), Callbacks(0), MacroArgCache(0),
     66     Record(0), MIChainHead(0), MICache(0)
     67 {
     68   OwnsHeaderSearch = OwnsHeaders;
     69 
     70   if (!DelayInitialization) {
     71     assert(Target && "Must provide target information for PP initialization");
     72     Initialize(*Target);
     73   }
     74 }
     75 
     76 Preprocessor::~Preprocessor() {
     77   assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
     78 
     79   while (!IncludeMacroStack.empty()) {
     80     delete IncludeMacroStack.back().TheLexer;
     81     delete IncludeMacroStack.back().TheTokenLexer;
     82     IncludeMacroStack.pop_back();
     83   }
     84 
     85   // Free any macro definitions.
     86   for (MacroInfoChain *I = MIChainHead ; I ; I = I->Next)
     87     I->MI.Destroy();
     88 
     89   // Free any cached macro expanders.
     90   for (unsigned i = 0, e = NumCachedTokenLexers; i != e; ++i)
     91     delete TokenLexerCache[i];
     92 
     93   // Free any cached MacroArgs.
     94   for (MacroArgs *ArgList = MacroArgCache; ArgList; )
     95     ArgList = ArgList->deallocate();
     96 
     97   // Release pragma information.
     98   delete PragmaHandlers;
     99 
    100   // Delete the scratch buffer info.
    101   delete ScratchBuf;
    102 
    103   // Delete the header search info, if we own it.
    104   if (OwnsHeaderSearch)
    105     delete &HeaderInfo;
    106 
    107   delete Callbacks;
    108 }
    109 
    110 void Preprocessor::Initialize(const TargetInfo &Target) {
    111   assert((!this->Target || this->Target == &Target) &&
    112          "Invalid override of target information");
    113   this->Target = &Target;
    114 
    115   // Initialize information about built-ins.
    116   BuiltinInfo.InitializeTarget(Target);
    117 
    118   ScratchBuf = new ScratchBuffer(SourceMgr);
    119   CounterValue = 0; // __COUNTER__ starts at 0.
    120 
    121   // Clear stats.
    122   NumDirectives = NumDefined = NumUndefined = NumPragma = 0;
    123   NumIf = NumElse = NumEndif = 0;
    124   NumEnteredSourceFiles = 0;
    125   NumMacroExpanded = NumFnMacroExpanded = NumBuiltinMacroExpanded = 0;
    126   NumFastMacroExpanded = NumTokenPaste = NumFastTokenPaste = 0;
    127   MaxIncludeStackDepth = 0;
    128   NumSkipped = 0;
    129 
    130   // Default to discarding comments.
    131   KeepComments = false;
    132   KeepMacroComments = false;
    133   SuppressIncludeNotFoundError = false;
    134 
    135   // Macro expansion is enabled.
    136   DisableMacroExpansion = false;
    137   InMacroArgs = false;
    138   InMacroArgPreExpansion = false;
    139   NumCachedTokenLexers = 0;
    140 
    141   CachedLexPos = 0;
    142 
    143   // We haven't read anything from the external source.
    144   ReadMacrosFromExternalSource = false;
    145 
    146   // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro.
    147   // This gets unpoisoned where it is allowed.
    148   (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
    149   SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
    150 
    151   // Initialize the pragma handlers.
    152   PragmaHandlers = new PragmaNamespace(StringRef());
    153   RegisterBuiltinPragmas();
    154 
    155   // Initialize builtin macros like __LINE__ and friends.
    156   RegisterBuiltinMacros();
    157 
    158   if(LangOpts.Borland) {
    159     Ident__exception_info        = getIdentifierInfo("_exception_info");
    160     Ident___exception_info       = getIdentifierInfo("__exception_info");
    161     Ident_GetExceptionInfo       = getIdentifierInfo("GetExceptionInformation");
    162     Ident__exception_code        = getIdentifierInfo("_exception_code");
    163     Ident___exception_code       = getIdentifierInfo("__exception_code");
    164     Ident_GetExceptionCode       = getIdentifierInfo("GetExceptionCode");
    165     Ident__abnormal_termination  = getIdentifierInfo("_abnormal_termination");
    166     Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
    167     Ident_AbnormalTermination    = getIdentifierInfo("AbnormalTermination");
    168   } else {
    169     Ident__exception_info = Ident__exception_code = Ident__abnormal_termination = 0;
    170     Ident___exception_info = Ident___exception_code = Ident___abnormal_termination = 0;
    171     Ident_GetExceptionInfo = Ident_GetExceptionCode = Ident_AbnormalTermination = 0;
    172   }
    173 
    174   HeaderInfo.setTarget(Target);
    175 }
    176 
    177 void Preprocessor::setPTHManager(PTHManager* pm) {
    178   PTH.reset(pm);
    179   FileMgr.addStatCache(PTH->createStatCache());
    180 }
    181 
    182 void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
    183   llvm::errs() << tok::getTokenName(Tok.getKind()) << " '"
    184                << getSpelling(Tok) << "'";
    185 
    186   if (!DumpFlags) return;
    187 
    188   llvm::errs() << "\t";
    189   if (Tok.isAtStartOfLine())
    190     llvm::errs() << " [StartOfLine]";
    191   if (Tok.hasLeadingSpace())
    192     llvm::errs() << " [LeadingSpace]";
    193   if (Tok.isExpandDisabled())
    194     llvm::errs() << " [ExpandDisabled]";
    195   if (Tok.needsCleaning()) {
    196     const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
    197     llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
    198                  << "']";
    199   }
    200 
    201   llvm::errs() << "\tLoc=<";
    202   DumpLocation(Tok.getLocation());
    203   llvm::errs() << ">";
    204 }
    205 
    206 void Preprocessor::DumpLocation(SourceLocation Loc) const {
    207   Loc.dump(SourceMgr);
    208 }
    209 
    210 void Preprocessor::DumpMacro(const MacroInfo &MI) const {
    211   llvm::errs() << "MACRO: ";
    212   for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
    213     DumpToken(MI.getReplacementToken(i));
    214     llvm::errs() << "  ";
    215   }
    216   llvm::errs() << "\n";
    217 }
    218 
    219 void Preprocessor::PrintStats() {
    220   llvm::errs() << "\n*** Preprocessor Stats:\n";
    221   llvm::errs() << NumDirectives << " directives found:\n";
    222   llvm::errs() << "  " << NumDefined << " #define.\n";
    223   llvm::errs() << "  " << NumUndefined << " #undef.\n";
    224   llvm::errs() << "  #include/#include_next/#import:\n";
    225   llvm::errs() << "    " << NumEnteredSourceFiles << " source files entered.\n";
    226   llvm::errs() << "    " << MaxIncludeStackDepth << " max include stack depth\n";
    227   llvm::errs() << "  " << NumIf << " #if/#ifndef/#ifdef.\n";
    228   llvm::errs() << "  " << NumElse << " #else/#elif.\n";
    229   llvm::errs() << "  " << NumEndif << " #endif.\n";
    230   llvm::errs() << "  " << NumPragma << " #pragma.\n";
    231   llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
    232 
    233   llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
    234              << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
    235              << NumFastMacroExpanded << " on the fast path.\n";
    236   llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
    237              << " token paste (##) operations performed, "
    238              << NumFastTokenPaste << " on the fast path.\n";
    239 }
    240 
    241 Preprocessor::macro_iterator
    242 Preprocessor::macro_begin(bool IncludeExternalMacros) const {
    243   if (IncludeExternalMacros && ExternalSource &&
    244       !ReadMacrosFromExternalSource) {
    245     ReadMacrosFromExternalSource = true;
    246     ExternalSource->ReadDefinedMacros();
    247   }
    248 
    249   return Macros.begin();
    250 }
    251 
    252 size_t Preprocessor::getTotalMemory() const {
    253   return BP.getTotalMemory()
    254     + llvm::capacity_in_bytes(MacroExpandedTokens)
    255     + Predefines.capacity() /* Predefines buffer. */
    256     + llvm::capacity_in_bytes(Macros)
    257     + llvm::capacity_in_bytes(PragmaPushMacroInfo)
    258     + llvm::capacity_in_bytes(PoisonReasons)
    259     + llvm::capacity_in_bytes(CommentHandlers);
    260 }
    261 
    262 Preprocessor::macro_iterator
    263 Preprocessor::macro_end(bool IncludeExternalMacros) const {
    264   if (IncludeExternalMacros && ExternalSource &&
    265       !ReadMacrosFromExternalSource) {
    266     ReadMacrosFromExternalSource = true;
    267     ExternalSource->ReadDefinedMacros();
    268   }
    269 
    270   return Macros.end();
    271 }
    272 
    273 void Preprocessor::recomputeCurLexerKind() {
    274   if (CurLexer)
    275     CurLexerKind = CLK_Lexer;
    276   else if (CurPTHLexer)
    277     CurLexerKind = CLK_PTHLexer;
    278   else if (CurTokenLexer)
    279     CurLexerKind = CLK_TokenLexer;
    280   else
    281     CurLexerKind = CLK_CachingLexer;
    282 }
    283 
    284 bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
    285                                           unsigned CompleteLine,
    286                                           unsigned CompleteColumn) {
    287   assert(File);
    288   assert(CompleteLine && CompleteColumn && "Starts from 1:1");
    289   assert(!CodeCompletionFile && "Already set");
    290 
    291   using llvm::MemoryBuffer;
    292 
    293   // Load the actual file's contents.
    294   bool Invalid = false;
    295   const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid);
    296   if (Invalid)
    297     return true;
    298 
    299   // Find the byte position of the truncation point.
    300   const char *Position = Buffer->getBufferStart();
    301   for (unsigned Line = 1; Line < CompleteLine; ++Line) {
    302     for (; *Position; ++Position) {
    303       if (*Position != '\r' && *Position != '\n')
    304         continue;
    305 
    306       // Eat \r\n or \n\r as a single line.
    307       if ((Position[1] == '\r' || Position[1] == '\n') &&
    308           Position[0] != Position[1])
    309         ++Position;
    310       ++Position;
    311       break;
    312     }
    313   }
    314 
    315   Position += CompleteColumn - 1;
    316 
    317   // Insert '\0' at the code-completion point.
    318   if (Position < Buffer->getBufferEnd()) {
    319     CodeCompletionFile = File;
    320     CodeCompletionOffset = Position - Buffer->getBufferStart();
    321 
    322     MemoryBuffer *NewBuffer =
    323         MemoryBuffer::getNewUninitMemBuffer(Buffer->getBufferSize() + 1,
    324                                             Buffer->getBufferIdentifier());
    325     char *NewBuf = const_cast<char*>(NewBuffer->getBufferStart());
    326     char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
    327     *NewPos = '\0';
    328     std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
    329     SourceMgr.overrideFileContents(File, NewBuffer);
    330   }
    331 
    332   return false;
    333 }
    334 
    335 void Preprocessor::CodeCompleteNaturalLanguage() {
    336   if (CodeComplete)
    337     CodeComplete->CodeCompleteNaturalLanguage();
    338   setCodeCompletionReached();
    339 }
    340 
    341 /// getSpelling - This method is used to get the spelling of a token into a
    342 /// SmallVector. Note that the returned StringRef may not point to the
    343 /// supplied buffer if a copy can be avoided.
    344 StringRef Preprocessor::getSpelling(const Token &Tok,
    345                                           SmallVectorImpl<char> &Buffer,
    346                                           bool *Invalid) const {
    347   // NOTE: this has to be checked *before* testing for an IdentifierInfo.
    348   if (Tok.isNot(tok::raw_identifier)) {
    349     // Try the fast path.
    350     if (const IdentifierInfo *II = Tok.getIdentifierInfo())
    351       return II->getName();
    352   }
    353 
    354   // Resize the buffer if we need to copy into it.
    355   if (Tok.needsCleaning())
    356     Buffer.resize(Tok.getLength());
    357 
    358   const char *Ptr = Buffer.data();
    359   unsigned Len = getSpelling(Tok, Ptr, Invalid);
    360   return StringRef(Ptr, Len);
    361 }
    362 
    363 /// CreateString - Plop the specified string into a scratch buffer and return a
    364 /// location for it.  If specified, the source location provides a source
    365 /// location for the token.
    366 void Preprocessor::CreateString(const char *Buf, unsigned Len, Token &Tok,
    367                                 SourceLocation ExpansionLocStart,
    368                                 SourceLocation ExpansionLocEnd) {
    369   Tok.setLength(Len);
    370 
    371   const char *DestPtr;
    372   SourceLocation Loc = ScratchBuf->getToken(Buf, Len, DestPtr);
    373 
    374   if (ExpansionLocStart.isValid())
    375     Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
    376                                        ExpansionLocEnd, Len);
    377   Tok.setLocation(Loc);
    378 
    379   // If this is a raw identifier or a literal token, set the pointer data.
    380   if (Tok.is(tok::raw_identifier))
    381     Tok.setRawIdentifierData(DestPtr);
    382   else if (Tok.isLiteral())
    383     Tok.setLiteralData(DestPtr);
    384 }
    385 
    386 Module *Preprocessor::getCurrentModule() {
    387   if (getLangOpts().CurrentModule.empty())
    388     return 0;
    389 
    390   return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule);
    391 }
    392 
    393 //===----------------------------------------------------------------------===//
    394 // Preprocessor Initialization Methods
    395 //===----------------------------------------------------------------------===//
    396 
    397 
    398 /// EnterMainSourceFile - Enter the specified FileID as the main source file,
    399 /// which implicitly adds the builtin defines etc.
    400 void Preprocessor::EnterMainSourceFile() {
    401   // We do not allow the preprocessor to reenter the main file.  Doing so will
    402   // cause FileID's to accumulate information from both runs (e.g. #line
    403   // information) and predefined macros aren't guaranteed to be set properly.
    404   assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
    405   FileID MainFileID = SourceMgr.getMainFileID();
    406 
    407   // If MainFileID is loaded it means we loaded an AST file, no need to enter
    408   // a main file.
    409   if (!SourceMgr.isLoadedFileID(MainFileID)) {
    410     // Enter the main file source buffer.
    411     EnterSourceFile(MainFileID, 0, SourceLocation());
    412 
    413     // If we've been asked to skip bytes in the main file (e.g., as part of a
    414     // precompiled preamble), do so now.
    415     if (SkipMainFilePreamble.first > 0)
    416       CurLexer->SkipBytes(SkipMainFilePreamble.first,
    417                           SkipMainFilePreamble.second);
    418 
    419     // Tell the header info that the main file was entered.  If the file is later
    420     // #imported, it won't be re-entered.
    421     if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
    422       HeaderInfo.IncrementIncludeCount(FE);
    423   }
    424 
    425   // Preprocess Predefines to populate the initial preprocessor state.
    426   llvm::MemoryBuffer *SB =
    427     llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
    428   assert(SB && "Cannot create predefined source buffer");
    429   FileID FID = SourceMgr.createFileIDForMemBuffer(SB);
    430   assert(!FID.isInvalid() && "Could not create FileID for predefines?");
    431 
    432   // Start parsing the predefines.
    433   EnterSourceFile(FID, 0, SourceLocation());
    434 }
    435 
    436 void Preprocessor::EndSourceFile() {
    437   // Notify the client that we reached the end of the source file.
    438   if (Callbacks)
    439     Callbacks->EndOfMainFile();
    440 }
    441 
    442 //===----------------------------------------------------------------------===//
    443 // Lexer Event Handling.
    444 //===----------------------------------------------------------------------===//
    445 
    446 /// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
    447 /// identifier information for the token and install it into the token,
    448 /// updating the token kind accordingly.
    449 IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const {
    450   assert(Identifier.getRawIdentifierData() != 0 && "No raw identifier data!");
    451 
    452   // Look up this token, see if it is a macro, or if it is a language keyword.
    453   IdentifierInfo *II;
    454   if (!Identifier.needsCleaning()) {
    455     // No cleaning needed, just use the characters from the lexed buffer.
    456     II = getIdentifierInfo(StringRef(Identifier.getRawIdentifierData(),
    457                                            Identifier.getLength()));
    458   } else {
    459     // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
    460     SmallString<64> IdentifierBuffer;
    461     StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
    462     II = getIdentifierInfo(CleanedStr);
    463   }
    464 
    465   // Update the token info (identifier info and appropriate token kind).
    466   Identifier.setIdentifierInfo(II);
    467   Identifier.setKind(II->getTokenID());
    468 
    469   return II;
    470 }
    471 
    472 void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
    473   PoisonReasons[II] = DiagID;
    474 }
    475 
    476 void Preprocessor::PoisonSEHIdentifiers(bool Poison) {
    477   assert(Ident__exception_code && Ident__exception_info);
    478   assert(Ident___exception_code && Ident___exception_info);
    479   Ident__exception_code->setIsPoisoned(Poison);
    480   Ident___exception_code->setIsPoisoned(Poison);
    481   Ident_GetExceptionCode->setIsPoisoned(Poison);
    482   Ident__exception_info->setIsPoisoned(Poison);
    483   Ident___exception_info->setIsPoisoned(Poison);
    484   Ident_GetExceptionInfo->setIsPoisoned(Poison);
    485   Ident__abnormal_termination->setIsPoisoned(Poison);
    486   Ident___abnormal_termination->setIsPoisoned(Poison);
    487   Ident_AbnormalTermination->setIsPoisoned(Poison);
    488 }
    489 
    490 void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) {
    491   assert(Identifier.getIdentifierInfo() &&
    492          "Can't handle identifiers without identifier info!");
    493   llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
    494     PoisonReasons.find(Identifier.getIdentifierInfo());
    495   if(it == PoisonReasons.end())
    496     Diag(Identifier, diag::err_pp_used_poisoned_id);
    497   else
    498     Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
    499 }
    500 
    501 /// HandleIdentifier - This callback is invoked when the lexer reads an
    502 /// identifier.  This callback looks up the identifier in the map and/or
    503 /// potentially macro expands it or turns it into a named token (like 'for').
    504 ///
    505 /// Note that callers of this method are guarded by checking the
    506 /// IdentifierInfo's 'isHandleIdentifierCase' bit.  If this method changes, the
    507 /// IdentifierInfo methods that compute these properties will need to change to
    508 /// match.
    509 void Preprocessor::HandleIdentifier(Token &Identifier) {
    510   assert(Identifier.getIdentifierInfo() &&
    511          "Can't handle identifiers without identifier info!");
    512 
    513   IdentifierInfo &II = *Identifier.getIdentifierInfo();
    514 
    515   // If the information about this identifier is out of date, update it from
    516   // the external source.
    517   if (II.isOutOfDate()) {
    518     ExternalSource->updateOutOfDateIdentifier(II);
    519     Identifier.setKind(II.getTokenID());
    520   }
    521 
    522   // If this identifier was poisoned, and if it was not produced from a macro
    523   // expansion, emit an error.
    524   if (II.isPoisoned() && CurPPLexer) {
    525     HandlePoisonedIdentifier(Identifier);
    526   }
    527 
    528   // If this is a macro to be expanded, do it.
    529   if (MacroInfo *MI = getMacroInfo(&II)) {
    530     if (!DisableMacroExpansion) {
    531       if (Identifier.isExpandDisabled()) {
    532         Diag(Identifier, diag::pp_disabled_macro_expansion);
    533       } else if (MI->isEnabled()) {
    534         if (!HandleMacroExpandedIdentifier(Identifier, MI))
    535           return;
    536       } else {
    537         // C99 6.10.3.4p2 says that a disabled macro may never again be
    538         // expanded, even if it's in a context where it could be expanded in the
    539         // future.
    540         Identifier.setFlag(Token::DisableExpand);
    541         Diag(Identifier, diag::pp_disabled_macro_expansion);
    542       }
    543     }
    544   }
    545 
    546   // If this identifier is a keyword in C++11, produce a warning. Don't warn if
    547   // we're not considering macro expansion, since this identifier might be the
    548   // name of a macro.
    549   // FIXME: This warning is disabled in cases where it shouldn't be, like
    550   //   "#define constexpr constexpr", "int constexpr;"
    551   if (II.isCXX11CompatKeyword() & !DisableMacroExpansion) {
    552     Diag(Identifier, diag::warn_cxx11_keyword) << II.getName();
    553     // Don't diagnose this keyword again in this translation unit.
    554     II.setIsCXX11CompatKeyword(false);
    555   }
    556 
    557   // C++ 2.11p2: If this is an alternative representation of a C++ operator,
    558   // then we act as if it is the actual operator and not the textual
    559   // representation of it.
    560   if (II.isCPlusPlusOperatorKeyword())
    561     Identifier.setIdentifierInfo(0);
    562 
    563   // If this is an extension token, diagnose its use.
    564   // We avoid diagnosing tokens that originate from macro definitions.
    565   // FIXME: This warning is disabled in cases where it shouldn't be,
    566   // like "#define TY typeof", "TY(1) x".
    567   if (II.isExtensionToken() && !DisableMacroExpansion)
    568     Diag(Identifier, diag::ext_token_used);
    569 
    570   // If this is the '__experimental_modules_import' contextual keyword, note
    571   // that the next token indicates a module name.
    572   //
    573   // Note that we do not treat '__experimental_modules_import' as a contextual
    574   // keyword when we're in a caching lexer, because caching lexers only get
    575   // used in contexts where import declarations are disallowed.
    576   if (II.isModulesImport() && !InMacroArgs && !DisableMacroExpansion &&
    577       getLangOpts().Modules && CurLexerKind != CLK_CachingLexer) {
    578     ModuleImportLoc = Identifier.getLocation();
    579     ModuleImportPath.clear();
    580     ModuleImportExpectsIdentifier = true;
    581     CurLexerKind = CLK_LexAfterModuleImport;
    582   }
    583 }
    584 
    585 /// \brief Lex a token following the 'import' contextual keyword.
    586 ///
    587 void Preprocessor::LexAfterModuleImport(Token &Result) {
    588   // Figure out what kind of lexer we actually have.
    589   recomputeCurLexerKind();
    590 
    591   // Lex the next token.
    592   Lex(Result);
    593 
    594   // The token sequence
    595   //
    596   //   import identifier (. identifier)*
    597   //
    598   // indicates a module import directive. We already saw the 'import'
    599   // contextual keyword, so now we're looking for the identifiers.
    600   if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
    601     // We expected to see an identifier here, and we did; continue handling
    602     // identifiers.
    603     ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(),
    604                                               Result.getLocation()));
    605     ModuleImportExpectsIdentifier = false;
    606     CurLexerKind = CLK_LexAfterModuleImport;
    607     return;
    608   }
    609 
    610   // If we're expecting a '.' or a ';', and we got a '.', then wait until we
    611   // see the next identifier.
    612   if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
    613     ModuleImportExpectsIdentifier = true;
    614     CurLexerKind = CLK_LexAfterModuleImport;
    615     return;
    616   }
    617 
    618   // If we have a non-empty module path, load the named module.
    619   if (!ModuleImportPath.empty())
    620     (void)TheModuleLoader.loadModule(ModuleImportLoc, ModuleImportPath,
    621                                      Module::MacrosVisible,
    622                                      /*IsIncludeDirective=*/false);
    623 }
    624 
    625 void Preprocessor::AddCommentHandler(CommentHandler *Handler) {
    626   assert(Handler && "NULL comment handler");
    627   assert(std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler) ==
    628          CommentHandlers.end() && "Comment handler already registered");
    629   CommentHandlers.push_back(Handler);
    630 }
    631 
    632 void Preprocessor::RemoveCommentHandler(CommentHandler *Handler) {
    633   std::vector<CommentHandler *>::iterator Pos
    634   = std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler);
    635   assert(Pos != CommentHandlers.end() && "Comment handler not registered");
    636   CommentHandlers.erase(Pos);
    637 }
    638 
    639 bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
    640   bool AnyPendingTokens = false;
    641   for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
    642        HEnd = CommentHandlers.end();
    643        H != HEnd; ++H) {
    644     if ((*H)->HandleComment(*this, Comment))
    645       AnyPendingTokens = true;
    646   }
    647   if (!AnyPendingTokens || getCommentRetentionState())
    648     return false;
    649   Lex(result);
    650   return true;
    651 }
    652 
    653 ModuleLoader::~ModuleLoader() { }
    654 
    655 CommentHandler::~CommentHandler() { }
    656 
    657 CodeCompletionHandler::~CodeCompletionHandler() { }
    658 
    659 void Preprocessor::createPreprocessingRecord(bool RecordConditionalDirectives) {
    660   if (Record)
    661     return;
    662 
    663   Record = new PreprocessingRecord(getSourceManager(),
    664                                    RecordConditionalDirectives);
    665   addPPCallbacks(Record);
    666 }
    667