Home | History | Annotate | Download | only in Lex 
      1 //===--- Preprocess.cpp - C Language Family Preprocessor Implementation ---===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 //  This file implements the Preprocessor interface.
     11 //
     12 //===----------------------------------------------------------------------===//
     13 //
     14 // Options to support:
     15 //   -H       - Print the name of each header file used.
     16 //   -d[DNI] - Dump various things.
     17 //   -fworking-directory - #line's with preprocessor's working dir.
     18 //   -fpreprocessed
     19 //   -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
     20 //   -W*
     21 //   -w
     22 //
     23 // Messages to emit:
     24 //   "Multiple include guards may be useful for:\n"
     25 //
     26 //===----------------------------------------------------------------------===//
     27 
     28 #include "clang/Lex/Preprocessor.h"
     29 #include "MacroArgs.h"
     30 #include "clang/Lex/ExternalPreprocessorSource.h"
     31 #include "clang/Lex/HeaderSearch.h"
     32 #include "clang/Lex/MacroInfo.h"
     33 #include "clang/Lex/Pragma.h"
     34 #include "clang/Lex/PreprocessingRecord.h"
     35 #include "clang/Lex/ScratchBuffer.h"
     36 #include "clang/Lex/LexDiagnostic.h"
     37 #include "clang/Lex/CodeCompletionHandler.h"
     38 #include "clang/Lex/ModuleLoader.h"
     39 #include "clang/Basic/SourceManager.h"
     40 #include "clang/Basic/FileManager.h"
     41 #include "clang/Basic/TargetInfo.h"
     42 #include "llvm/ADT/APFloat.h"
     43 #include "llvm/ADT/SmallVector.h"
     44 #include "llvm/Support/MemoryBuffer.h"
     45 #include "llvm/Support/raw_ostream.h"
     46 #include "llvm/Support/Capacity.h"
     47 using namespace clang;
     48 
     49 //===----------------------------------------------------------------------===//
     50 ExternalPreprocessorSource::~ExternalPreprocessorSource() { }
     51 
     52 Preprocessor::Preprocessor(DiagnosticsEngine &diags, LangOptions &opts,
     53                            const TargetInfo *target, SourceManager &SM,
     54                            HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
     55                            IdentifierInfoLookup* IILookup,
     56                            bool OwnsHeaders,
     57                            bool DelayInitialization)
     58   : Diags(&diags), Features(opts), Target(target),FileMgr(Headers.getFileMgr()),
     59     SourceMgr(SM), HeaderInfo(Headers), TheModuleLoader(TheModuleLoader),
     60     ExternalSource(0),
     61     Identifiers(opts, IILookup), CodeComplete(0),
     62     CodeCompletionFile(0), CodeCompletionOffset(0), CodeCompletionReached(0),
     63     SkipMainFilePreamble(0, true), CurPPLexer(0),
     64     CurDirLookup(0), CurLexerKind(CLK_Lexer), Callbacks(0), MacroArgCache(0),
     65     Record(0), MIChainHead(0), MICache(0)
     66 {
     67   OwnsHeaderSearch = OwnsHeaders;
     68 
     69   if (!DelayInitialization) {
     70     assert(Target && "Must provide target information for PP initialization");
     71     Initialize(*Target);
     72   }
     73 }
     74 
     75 Preprocessor::~Preprocessor() {
     76   assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
     77   assert(((MacroExpandingLexersStack.empty() && MacroExpandedTokens.empty()) ||
     78           isCodeCompletionReached()) &&
     79          "Preprocessor::HandleEndOfTokenLexer should have cleared those");
     80 
     81   while (!IncludeMacroStack.empty()) {
     82     delete IncludeMacroStack.back().TheLexer;
     83     delete IncludeMacroStack.back().TheTokenLexer;
     84     IncludeMacroStack.pop_back();
     85   }
     86 
     87   // Free any macro definitions.
     88   for (MacroInfoChain *I = MIChainHead ; I ; I = I->Next)
     89     I->MI.Destroy();
     90 
     91   // Free any cached macro expanders.
     92   for (unsigned i = 0, e = NumCachedTokenLexers; i != e; ++i)
     93     delete TokenLexerCache[i];
     94 
     95   // Free any cached MacroArgs.
     96   for (MacroArgs *ArgList = MacroArgCache; ArgList; )
     97     ArgList = ArgList->deallocate();
     98 
     99   // Release pragma information.
    100   delete PragmaHandlers;
    101 
    102   // Delete the scratch buffer info.
    103   delete ScratchBuf;
    104 
    105   // Delete the header search info, if we own it.
    106   if (OwnsHeaderSearch)
    107     delete &HeaderInfo;
    108 
    109   delete Callbacks;
    110 }
    111 
    112 void Preprocessor::Initialize(const TargetInfo &Target) {
    113   assert((!this->Target || this->Target == &Target) &&
    114          "Invalid override of target information");
    115   this->Target = &Target;
    116 
    117   // Initialize information about built-ins.
    118   BuiltinInfo.InitializeTarget(Target);
    119 
    120   ScratchBuf = new ScratchBuffer(SourceMgr);
    121   CounterValue = 0; // __COUNTER__ starts at 0.
    122 
    123   // Clear stats.
    124   NumDirectives = NumDefined = NumUndefined = NumPragma = 0;
    125   NumIf = NumElse = NumEndif = 0;
    126   NumEnteredSourceFiles = 0;
    127   NumMacroExpanded = NumFnMacroExpanded = NumBuiltinMacroExpanded = 0;
    128   NumFastMacroExpanded = NumTokenPaste = NumFastTokenPaste = 0;
    129   MaxIncludeStackDepth = 0;
    130   NumSkipped = 0;
    131 
    132   // Default to discarding comments.
    133   KeepComments = false;
    134   KeepMacroComments = false;
    135   SuppressIncludeNotFoundError = false;
    136   AutoModuleImport = false;
    137 
    138   // Macro expansion is enabled.
    139   DisableMacroExpansion = false;
    140   InMacroArgs = false;
    141   NumCachedTokenLexers = 0;
    142 
    143   CachedLexPos = 0;
    144 
    145   // We haven't read anything from the external source.
    146   ReadMacrosFromExternalSource = false;
    147 
    148   // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro.
    149   // This gets unpoisoned where it is allowed.
    150   (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
    151   SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
    152 
    153   // Initialize the pragma handlers.
    154   PragmaHandlers = new PragmaNamespace(StringRef());
    155   RegisterBuiltinPragmas();
    156 
    157   // Initialize builtin macros like __LINE__ and friends.
    158   RegisterBuiltinMacros();
    159 
    160   if(Features.Borland) {
    161     Ident__exception_info        = getIdentifierInfo("_exception_info");
    162     Ident___exception_info       = getIdentifierInfo("__exception_info");
    163     Ident_GetExceptionInfo       = getIdentifierInfo("GetExceptionInformation");
    164     Ident__exception_code        = getIdentifierInfo("_exception_code");
    165     Ident___exception_code       = getIdentifierInfo("__exception_code");
    166     Ident_GetExceptionCode       = getIdentifierInfo("GetExceptionCode");
    167     Ident__abnormal_termination  = getIdentifierInfo("_abnormal_termination");
    168     Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
    169     Ident_AbnormalTermination    = getIdentifierInfo("AbnormalTermination");
    170   } else {
    171     Ident__exception_info = Ident__exception_code = Ident__abnormal_termination = 0;
    172     Ident___exception_info = Ident___exception_code = Ident___abnormal_termination = 0;
    173     Ident_GetExceptionInfo = Ident_GetExceptionCode = Ident_AbnormalTermination = 0;
    174   }
    175 }
    176 
    177 void Preprocessor::setPTHManager(PTHManager* pm) {
    178   PTH.reset(pm);
    179   FileMgr.addStatCache(PTH->createStatCache());
    180 }
    181 
    182 void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
    183   llvm::errs() << tok::getTokenName(Tok.getKind()) << " '"
    184                << getSpelling(Tok) << "'";
    185 
    186   if (!DumpFlags) return;
    187 
    188   llvm::errs() << "\t";
    189   if (Tok.isAtStartOfLine())
    190     llvm::errs() << " [StartOfLine]";
    191   if (Tok.hasLeadingSpace())
    192     llvm::errs() << " [LeadingSpace]";
    193   if (Tok.isExpandDisabled())
    194     llvm::errs() << " [ExpandDisabled]";
    195   if (Tok.needsCleaning()) {
    196     const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
    197     llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
    198                  << "']";
    199   }
    200 
    201   llvm::errs() << "\tLoc=<";
    202   DumpLocation(Tok.getLocation());
    203   llvm::errs() << ">";
    204 }
    205 
    206 void Preprocessor::DumpLocation(SourceLocation Loc) const {
    207   Loc.dump(SourceMgr);
    208 }
    209 
    210 void Preprocessor::DumpMacro(const MacroInfo &MI) const {
    211   llvm::errs() << "MACRO: ";
    212   for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
    213     DumpToken(MI.getReplacementToken(i));
    214     llvm::errs() << "  ";
    215   }
    216   llvm::errs() << "\n";
    217 }
    218 
    219 void Preprocessor::PrintStats() {
    220   llvm::errs() << "\n*** Preprocessor Stats:\n";
    221   llvm::errs() << NumDirectives << " directives found:\n";
    222   llvm::errs() << "  " << NumDefined << " #define.\n";
    223   llvm::errs() << "  " << NumUndefined << " #undef.\n";
    224   llvm::errs() << "  #include/#include_next/#import:\n";
    225   llvm::errs() << "    " << NumEnteredSourceFiles << " source files entered.\n";
    226   llvm::errs() << "    " << MaxIncludeStackDepth << " max include stack depth\n";
    227   llvm::errs() << "  " << NumIf << " #if/#ifndef/#ifdef.\n";
    228   llvm::errs() << "  " << NumElse << " #else/#elif.\n";
    229   llvm::errs() << "  " << NumEndif << " #endif.\n";
    230   llvm::errs() << "  " << NumPragma << " #pragma.\n";
    231   llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
    232 
    233   llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
    234              << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
    235              << NumFastMacroExpanded << " on the fast path.\n";
    236   llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
    237              << " token paste (##) operations performed, "
    238              << NumFastTokenPaste << " on the fast path.\n";
    239 }
    240 
    241 Preprocessor::macro_iterator
    242 Preprocessor::macro_begin(bool IncludeExternalMacros) const {
    243   if (IncludeExternalMacros && ExternalSource &&
    244       !ReadMacrosFromExternalSource) {
    245     ReadMacrosFromExternalSource = true;
    246     ExternalSource->ReadDefinedMacros();
    247   }
    248 
    249   return Macros.begin();
    250 }
    251 
    252 size_t Preprocessor::getTotalMemory() const {
    253   return BP.getTotalMemory()
    254     + llvm::capacity_in_bytes(MacroExpandedTokens)
    255     + Predefines.capacity() /* Predefines buffer. */
    256     + llvm::capacity_in_bytes(Macros)
    257     + llvm::capacity_in_bytes(PragmaPushMacroInfo)
    258     + llvm::capacity_in_bytes(PoisonReasons)
    259     + llvm::capacity_in_bytes(CommentHandlers);
    260 }
    261 
    262 Preprocessor::macro_iterator
    263 Preprocessor::macro_end(bool IncludeExternalMacros) const {
    264   if (IncludeExternalMacros && ExternalSource &&
    265       !ReadMacrosFromExternalSource) {
    266     ReadMacrosFromExternalSource = true;
    267     ExternalSource->ReadDefinedMacros();
    268   }
    269 
    270   return Macros.end();
    271 }
    272 
    273 bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
    274                                           unsigned CompleteLine,
    275                                           unsigned CompleteColumn) {
    276   assert(File);
    277   assert(CompleteLine && CompleteColumn && "Starts from 1:1");
    278   assert(!CodeCompletionFile && "Already set");
    279 
    280   using llvm::MemoryBuffer;
    281 
    282   // Load the actual file's contents.
    283   bool Invalid = false;
    284   const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid);
    285   if (Invalid)
    286     return true;
    287 
    288   // Find the byte position of the truncation point.
    289   const char *Position = Buffer->getBufferStart();
    290   for (unsigned Line = 1; Line < CompleteLine; ++Line) {
    291     for (; *Position; ++Position) {
    292       if (*Position != '\r' && *Position != '\n')
    293         continue;
    294 
    295       // Eat \r\n or \n\r as a single line.
    296       if ((Position[1] == '\r' || Position[1] == '\n') &&
    297           Position[0] != Position[1])
    298         ++Position;
    299       ++Position;
    300       break;
    301     }
    302   }
    303 
    304   Position += CompleteColumn - 1;
    305 
    306   // Insert '\0' at the code-completion point.
    307   if (Position < Buffer->getBufferEnd()) {
    308     CodeCompletionFile = File;
    309     CodeCompletionOffset = Position - Buffer->getBufferStart();
    310 
    311     MemoryBuffer *NewBuffer =
    312         MemoryBuffer::getNewUninitMemBuffer(Buffer->getBufferSize() + 1,
    313                                             Buffer->getBufferIdentifier());
    314     char *NewBuf = const_cast<char*>(NewBuffer->getBufferStart());
    315     char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
    316     *NewPos = '\0';
    317     std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
    318     SourceMgr.overrideFileContents(File, NewBuffer);
    319   }
    320 
    321   return false;
    322 }
    323 
    324 void Preprocessor::CodeCompleteNaturalLanguage() {
    325   if (CodeComplete)
    326     CodeComplete->CodeCompleteNaturalLanguage();
    327   setCodeCompletionReached();
    328 }
    329 
    330 /// getSpelling - This method is used to get the spelling of a token into a
    331 /// SmallVector. Note that the returned StringRef may not point to the
    332 /// supplied buffer if a copy can be avoided.
    333 StringRef Preprocessor::getSpelling(const Token &Tok,
    334                                           SmallVectorImpl<char> &Buffer,
    335                                           bool *Invalid) const {
    336   // NOTE: this has to be checked *before* testing for an IdentifierInfo.
    337   if (Tok.isNot(tok::raw_identifier)) {
    338     // Try the fast path.
    339     if (const IdentifierInfo *II = Tok.getIdentifierInfo())
    340       return II->getName();
    341   }
    342 
    343   // Resize the buffer if we need to copy into it.
    344   if (Tok.needsCleaning())
    345     Buffer.resize(Tok.getLength());
    346 
    347   const char *Ptr = Buffer.data();
    348   unsigned Len = getSpelling(Tok, Ptr, Invalid);
    349   return StringRef(Ptr, Len);
    350 }
    351 
    352 /// CreateString - Plop the specified string into a scratch buffer and return a
    353 /// location for it.  If specified, the source location provides a source
    354 /// location for the token.
    355 void Preprocessor::CreateString(const char *Buf, unsigned Len, Token &Tok,
    356                                 SourceLocation ExpansionLocStart,
    357                                 SourceLocation ExpansionLocEnd) {
    358   Tok.setLength(Len);
    359 
    360   const char *DestPtr;
    361   SourceLocation Loc = ScratchBuf->getToken(Buf, Len, DestPtr);
    362 
    363   if (ExpansionLocStart.isValid())
    364     Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
    365                                        ExpansionLocEnd, Len);
    366   Tok.setLocation(Loc);
    367 
    368   // If this is a raw identifier or a literal token, set the pointer data.
    369   if (Tok.is(tok::raw_identifier))
    370     Tok.setRawIdentifierData(DestPtr);
    371   else if (Tok.isLiteral())
    372     Tok.setLiteralData(DestPtr);
    373 }
    374 
    375 
    376 
    377 //===----------------------------------------------------------------------===//
    378 // Preprocessor Initialization Methods
    379 //===----------------------------------------------------------------------===//
    380 
    381 
    382 /// EnterMainSourceFile - Enter the specified FileID as the main source file,
    383 /// which implicitly adds the builtin defines etc.
    384 void Preprocessor::EnterMainSourceFile() {
    385   // We do not allow the preprocessor to reenter the main file.  Doing so will
    386   // cause FileID's to accumulate information from both runs (e.g. #line
    387   // information) and predefined macros aren't guaranteed to be set properly.
    388   assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
    389   FileID MainFileID = SourceMgr.getMainFileID();
    390 
    391   // Enter the main file source buffer.
    392   EnterSourceFile(MainFileID, 0, SourceLocation());
    393 
    394   // If we've been asked to skip bytes in the main file (e.g., as part of a
    395   // precompiled preamble), do so now.
    396   if (SkipMainFilePreamble.first > 0)
    397     CurLexer->SkipBytes(SkipMainFilePreamble.first,
    398                         SkipMainFilePreamble.second);
    399 
    400   // Tell the header info that the main file was entered.  If the file is later
    401   // #imported, it won't be re-entered.
    402   if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
    403     HeaderInfo.IncrementIncludeCount(FE);
    404 
    405   // Preprocess Predefines to populate the initial preprocessor state.
    406   llvm::MemoryBuffer *SB =
    407     llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
    408   assert(SB && "Cannot create predefined source buffer");
    409   FileID FID = SourceMgr.createFileIDForMemBuffer(SB);
    410   assert(!FID.isInvalid() && "Could not create FileID for predefines?");
    411 
    412   // Start parsing the predefines.
    413   EnterSourceFile(FID, 0, SourceLocation());
    414 }
    415 
    416 void Preprocessor::EndSourceFile() {
    417   // Notify the client that we reached the end of the source file.
    418   if (Callbacks)
    419     Callbacks->EndOfMainFile();
    420 }
    421 
    422 //===----------------------------------------------------------------------===//
    423 // Lexer Event Handling.
    424 //===----------------------------------------------------------------------===//
    425 
    426 /// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
    427 /// identifier information for the token and install it into the token,
    428 /// updating the token kind accordingly.
    429 IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const {
    430   assert(Identifier.getRawIdentifierData() != 0 && "No raw identifier data!");
    431 
    432   // Look up this token, see if it is a macro, or if it is a language keyword.
    433   IdentifierInfo *II;
    434   if (!Identifier.needsCleaning()) {
    435     // No cleaning needed, just use the characters from the lexed buffer.
    436     II = getIdentifierInfo(StringRef(Identifier.getRawIdentifierData(),
    437                                            Identifier.getLength()));
    438   } else {
    439     // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
    440     llvm::SmallString<64> IdentifierBuffer;
    441     StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
    442     II = getIdentifierInfo(CleanedStr);
    443   }
    444 
    445   // Update the token info (identifier info and appropriate token kind).
    446   Identifier.setIdentifierInfo(II);
    447   Identifier.setKind(II->getTokenID());
    448 
    449   return II;
    450 }
    451 
    452 void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
    453   PoisonReasons[II] = DiagID;
    454 }
    455 
    456 void Preprocessor::PoisonSEHIdentifiers(bool Poison) {
    457   assert(Ident__exception_code && Ident__exception_info);
    458   assert(Ident___exception_code && Ident___exception_info);
    459   Ident__exception_code->setIsPoisoned(Poison);
    460   Ident___exception_code->setIsPoisoned(Poison);
    461   Ident_GetExceptionCode->setIsPoisoned(Poison);
    462   Ident__exception_info->setIsPoisoned(Poison);
    463   Ident___exception_info->setIsPoisoned(Poison);
    464   Ident_GetExceptionInfo->setIsPoisoned(Poison);
    465   Ident__abnormal_termination->setIsPoisoned(Poison);
    466   Ident___abnormal_termination->setIsPoisoned(Poison);
    467   Ident_AbnormalTermination->setIsPoisoned(Poison);
    468 }
    469 
    470 void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) {
    471   assert(Identifier.getIdentifierInfo() &&
    472          "Can't handle identifiers without identifier info!");
    473   llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
    474     PoisonReasons.find(Identifier.getIdentifierInfo());
    475   if(it == PoisonReasons.end())
    476     Diag(Identifier, diag::err_pp_used_poisoned_id);
    477   else
    478     Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
    479 }
    480 
    481 /// HandleIdentifier - This callback is invoked when the lexer reads an
    482 /// identifier.  This callback looks up the identifier in the map and/or
    483 /// potentially macro expands it or turns it into a named token (like 'for').
    484 ///
    485 /// Note that callers of this method are guarded by checking the
    486 /// IdentifierInfo's 'isHandleIdentifierCase' bit.  If this method changes, the
    487 /// IdentifierInfo methods that compute these properties will need to change to
    488 /// match.
    489 void Preprocessor::HandleIdentifier(Token &Identifier) {
    490   assert(Identifier.getIdentifierInfo() &&
    491          "Can't handle identifiers without identifier info!");
    492 
    493   IdentifierInfo &II = *Identifier.getIdentifierInfo();
    494 
    495   // If this identifier was poisoned, and if it was not produced from a macro
    496   // expansion, emit an error.
    497   if (II.isPoisoned() && CurPPLexer) {
    498     HandlePoisonedIdentifier(Identifier);
    499   }
    500 
    501   // If this is a macro to be expanded, do it.
    502   if (MacroInfo *MI = getMacroInfo(&II)) {
    503     if (!DisableMacroExpansion && !Identifier.isExpandDisabled()) {
    504       if (MI->isEnabled()) {
    505         if (!HandleMacroExpandedIdentifier(Identifier, MI))
    506           return;
    507       } else {
    508         // C99 6.10.3.4p2 says that a disabled macro may never again be
    509         // expanded, even if it's in a context where it could be expanded in the
    510         // future.
    511         Identifier.setFlag(Token::DisableExpand);
    512       }
    513     }
    514   }
    515 
    516   // If this identifier is a keyword in C++11, produce a warning. Don't warn if
    517   // we're not considering macro expansion, since this identifier might be the
    518   // name of a macro.
    519   // FIXME: This warning is disabled in cases where it shouldn't be, like
    520   //   "#define constexpr constexpr", "int constexpr;"
    521   if (II.isCXX11CompatKeyword() & !DisableMacroExpansion) {
    522     Diag(Identifier, diag::warn_cxx11_keyword) << II.getName();
    523     // Don't diagnose this keyword again in this translation unit.
    524     II.setIsCXX11CompatKeyword(false);
    525   }
    526 
    527   // C++ 2.11p2: If this is an alternative representation of a C++ operator,
    528   // then we act as if it is the actual operator and not the textual
    529   // representation of it.
    530   if (II.isCPlusPlusOperatorKeyword())
    531     Identifier.setIdentifierInfo(0);
    532 
    533   // If this is an extension token, diagnose its use.
    534   // We avoid diagnosing tokens that originate from macro definitions.
    535   // FIXME: This warning is disabled in cases where it shouldn't be,
    536   // like "#define TY typeof", "TY(1) x".
    537   if (II.isExtensionToken() && !DisableMacroExpansion)
    538     Diag(Identifier, diag::ext_token_used);
    539 
    540   // If this is the '__import_module__' keyword, note that the next token
    541   // indicates a module name.
    542   if (II.getTokenID() == tok::kw___import_module__ &&
    543       !InMacroArgs && !DisableMacroExpansion) {
    544     ModuleImportLoc = Identifier.getLocation();
    545     CurLexerKind = CLK_LexAfterModuleImport;
    546   }
    547 }
    548 
    549 /// \brief Lex a token following the __import_module__ keyword.
    550 void Preprocessor::LexAfterModuleImport(Token &Result) {
    551   // Figure out what kind of lexer we actually have.
    552   if (CurLexer)
    553     CurLexerKind = CLK_Lexer;
    554   else if (CurPTHLexer)
    555     CurLexerKind = CLK_PTHLexer;
    556   else if (CurTokenLexer)
    557     CurLexerKind = CLK_TokenLexer;
    558   else
    559     CurLexerKind = CLK_CachingLexer;
    560 
    561   // Lex the next token.
    562   Lex(Result);
    563 
    564   // The token sequence
    565   //
    566   //   __import_module__ identifier
    567   //
    568   // indicates a module import directive. We already saw the __import_module__
    569   // keyword, so now we're looking for the identifier.
    570   if (Result.getKind() != tok::identifier)
    571     return;
    572 
    573   // Load the module.
    574   (void)TheModuleLoader.loadModule(ModuleImportLoc,
    575                                    *Result.getIdentifierInfo(),
    576                                    Result.getLocation());
    577 }
    578 
    579 void Preprocessor::AddCommentHandler(CommentHandler *Handler) {
    580   assert(Handler && "NULL comment handler");
    581   assert(std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler) ==
    582          CommentHandlers.end() && "Comment handler already registered");
    583   CommentHandlers.push_back(Handler);
    584 }
    585 
    586 void Preprocessor::RemoveCommentHandler(CommentHandler *Handler) {
    587   std::vector<CommentHandler *>::iterator Pos
    588   = std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler);
    589   assert(Pos != CommentHandlers.end() && "Comment handler not registered");
    590   CommentHandlers.erase(Pos);
    591 }
    592 
    593 bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
    594   bool AnyPendingTokens = false;
    595   for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
    596        HEnd = CommentHandlers.end();
    597        H != HEnd; ++H) {
    598     if ((*H)->HandleComment(*this, Comment))
    599       AnyPendingTokens = true;
    600   }
    601   if (!AnyPendingTokens || getCommentRetentionState())
    602     return false;
    603   Lex(result);
    604   return true;
    605 }
    606 
    607 ModuleLoader::~ModuleLoader() { }
    608 
    609 CommentHandler::~CommentHandler() { }
    610 
    611 CodeCompletionHandler::~CodeCompletionHandler() { }
    612 
    613 void Preprocessor::createPreprocessingRecord(
    614                                       bool IncludeNestedMacroExpansions) {
    615   if (Record)
    616     return;
    617 
    618   Record = new PreprocessingRecord(getSourceManager(),
    619                                    IncludeNestedMacroExpansions);
    620   addPPCallbacks(Record);
    621 }
    622