1 //===--- Preprocessor.h - C Language Family Preprocessor --------*- C++ -*-===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 /// 10 /// \file 11 /// \brief Defines the clang::Preprocessor interface. 12 /// 13 //===----------------------------------------------------------------------===// 14 15 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H 16 #define LLVM_CLANG_LEX_PREPROCESSOR_H 17 18 #include "clang/Basic/Builtins.h" 19 #include "clang/Basic/Diagnostic.h" 20 #include "clang/Basic/IdentifierTable.h" 21 #include "clang/Basic/SourceLocation.h" 22 #include "clang/Lex/Lexer.h" 23 #include "clang/Lex/MacroInfo.h" 24 #include "clang/Lex/ModuleMap.h" 25 #include "clang/Lex/PPCallbacks.h" 26 #include "clang/Lex/PTHLexer.h" 27 #include "clang/Lex/TokenLexer.h" 28 #include "llvm/ADT/ArrayRef.h" 29 #include "llvm/ADT/DenseMap.h" 30 #include "llvm/ADT/IntrusiveRefCntPtr.h" 31 #include "llvm/ADT/SmallPtrSet.h" 32 #include "llvm/ADT/SmallVector.h" 33 #include "llvm/ADT/TinyPtrVector.h" 34 #include "llvm/Support/Allocator.h" 35 #include "llvm/Support/Registry.h" 36 #include <memory> 37 #include <vector> 38 39 namespace llvm { 40 template<unsigned InternalLen> class SmallString; 41 } 42 43 namespace clang { 44 45 class SourceManager; 46 class ExternalPreprocessorSource; 47 class FileManager; 48 class FileEntry; 49 class HeaderSearch; 50 class MemoryBufferCache; 51 class PragmaNamespace; 52 class PragmaHandler; 53 class CommentHandler; 54 class ScratchBuffer; 55 class TargetInfo; 56 class PPCallbacks; 57 class CodeCompletionHandler; 58 class DirectoryLookup; 59 class PreprocessingRecord; 60 class ModuleLoader; 61 class PTHManager; 62 class PreprocessorOptions; 63 64 /// \brief Stores token information for comparing actual tokens with 65 /// predefined values. Only handles simple tokens and identifiers. 66 class TokenValue { 67 tok::TokenKind Kind; 68 IdentifierInfo *II; 69 70 public: 71 TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) { 72 assert(Kind != tok::raw_identifier && "Raw identifiers are not supported."); 73 assert(Kind != tok::identifier && 74 "Identifiers should be created by TokenValue(IdentifierInfo *)"); 75 assert(!tok::isLiteral(Kind) && "Literals are not supported."); 76 assert(!tok::isAnnotation(Kind) && "Annotations are not supported."); 77 } 78 TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {} 79 bool operator==(const Token &Tok) const { 80 return Tok.getKind() == Kind && 81 (!II || II == Tok.getIdentifierInfo()); 82 } 83 }; 84 85 /// \brief Context in which macro name is used. 86 enum MacroUse { 87 MU_Other = 0, // other than #define or #undef 88 MU_Define = 1, // macro name specified in #define 89 MU_Undef = 2 // macro name specified in #undef 90 }; 91 92 /// \brief Engages in a tight little dance with the lexer to efficiently 93 /// preprocess tokens. 94 /// 95 /// Lexers know only about tokens within a single source file, and don't 96 /// know anything about preprocessor-level issues like the \#include stack, 97 /// token expansion, etc. 98 class Preprocessor { 99 std::shared_ptr<PreprocessorOptions> PPOpts; 100 DiagnosticsEngine *Diags; 101 LangOptions &LangOpts; 102 const TargetInfo *Target; 103 const TargetInfo *AuxTarget; 104 FileManager &FileMgr; 105 SourceManager &SourceMgr; 106 MemoryBufferCache &PCMCache; 107 std::unique_ptr<ScratchBuffer> ScratchBuf; 108 HeaderSearch &HeaderInfo; 109 ModuleLoader &TheModuleLoader; 110 111 /// \brief External source of macros. 112 ExternalPreprocessorSource *ExternalSource; 113 114 115 /// An optional PTHManager object used for getting tokens from 116 /// a token cache rather than lexing the original source file. 117 std::unique_ptr<PTHManager> PTH; 118 119 /// A BumpPtrAllocator object used to quickly allocate and release 120 /// objects internal to the Preprocessor. 121 llvm::BumpPtrAllocator BP; 122 123 /// Identifiers for builtin macros and other builtins. 124 IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__ 125 IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__ 126 IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__ 127 IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__ 128 IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__ 129 IdentifierInfo *Ident__COUNTER__; // __COUNTER__ 130 IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma 131 IdentifierInfo *Ident__identifier; // __identifier 132 IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__ 133 IdentifierInfo *Ident__has_feature; // __has_feature 134 IdentifierInfo *Ident__has_extension; // __has_extension 135 IdentifierInfo *Ident__has_builtin; // __has_builtin 136 IdentifierInfo *Ident__has_attribute; // __has_attribute 137 IdentifierInfo *Ident__has_include; // __has_include 138 IdentifierInfo *Ident__has_include_next; // __has_include_next 139 IdentifierInfo *Ident__has_warning; // __has_warning 140 IdentifierInfo *Ident__is_identifier; // __is_identifier 141 IdentifierInfo *Ident__building_module; // __building_module 142 IdentifierInfo *Ident__MODULE__; // __MODULE__ 143 IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute 144 IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute 145 146 SourceLocation DATELoc, TIMELoc; 147 unsigned CounterValue; // Next __COUNTER__ value. 148 149 enum { 150 /// \brief Maximum depth of \#includes. 151 MaxAllowedIncludeStackDepth = 200 152 }; 153 154 // State that is set before the preprocessor begins. 155 bool KeepComments : 1; 156 bool KeepMacroComments : 1; 157 bool SuppressIncludeNotFoundError : 1; 158 159 // State that changes while the preprocessor runs: 160 bool InMacroArgs : 1; // True if parsing fn macro invocation args. 161 162 /// Whether the preprocessor owns the header search object. 163 bool OwnsHeaderSearch : 1; 164 165 /// True if macro expansion is disabled. 166 bool DisableMacroExpansion : 1; 167 168 /// Temporarily disables DisableMacroExpansion (i.e. enables expansion) 169 /// when parsing preprocessor directives. 170 bool MacroExpansionInDirectivesOverride : 1; 171 172 class ResetMacroExpansionHelper; 173 174 /// \brief Whether we have already loaded macros from the external source. 175 mutable bool ReadMacrosFromExternalSource : 1; 176 177 /// \brief True if pragmas are enabled. 178 bool PragmasEnabled : 1; 179 180 /// \brief True if the current build action is a preprocessing action. 181 bool PreprocessedOutput : 1; 182 183 /// \brief True if we are currently preprocessing a #if or #elif directive 184 bool ParsingIfOrElifDirective; 185 186 /// \brief True if we are pre-expanding macro arguments. 187 bool InMacroArgPreExpansion; 188 189 /// \brief Mapping/lookup information for all identifiers in 190 /// the program, including program keywords. 191 mutable IdentifierTable Identifiers; 192 193 /// \brief This table contains all the selectors in the program. 194 /// 195 /// Unlike IdentifierTable above, this table *isn't* populated by the 196 /// preprocessor. It is declared/expanded here because its role/lifetime is 197 /// conceptually similar to the IdentifierTable. In addition, the current 198 /// control flow (in clang::ParseAST()), make it convenient to put here. 199 /// 200 /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to 201 /// the lifetime of the preprocessor. 202 SelectorTable Selectors; 203 204 /// \brief Information about builtins. 205 Builtin::Context BuiltinInfo; 206 207 /// \brief Tracks all of the pragmas that the client registered 208 /// with this preprocessor. 209 std::unique_ptr<PragmaNamespace> PragmaHandlers; 210 211 /// \brief Pragma handlers of the original source is stored here during the 212 /// parsing of a model file. 213 std::unique_ptr<PragmaNamespace> PragmaHandlersBackup; 214 215 /// \brief Tracks all of the comment handlers that the client registered 216 /// with this preprocessor. 217 std::vector<CommentHandler *> CommentHandlers; 218 219 /// \brief True if we want to ignore EOF token and continue later on (thus 220 /// avoid tearing the Lexer and etc. down). 221 bool IncrementalProcessing; 222 223 /// The kind of translation unit we are processing. 224 TranslationUnitKind TUKind; 225 226 /// \brief The code-completion handler. 227 CodeCompletionHandler *CodeComplete; 228 229 /// \brief The file that we're performing code-completion for, if any. 230 const FileEntry *CodeCompletionFile; 231 232 /// \brief The offset in file for the code-completion point. 233 unsigned CodeCompletionOffset; 234 235 /// \brief The location for the code-completion point. This gets instantiated 236 /// when the CodeCompletionFile gets \#include'ed for preprocessing. 237 SourceLocation CodeCompletionLoc; 238 239 /// \brief The start location for the file of the code-completion point. 240 /// 241 /// This gets instantiated when the CodeCompletionFile gets \#include'ed 242 /// for preprocessing. 243 SourceLocation CodeCompletionFileLoc; 244 245 /// \brief The source location of the \c import contextual keyword we just 246 /// lexed, if any. 247 SourceLocation ModuleImportLoc; 248 249 /// \brief The module import path that we're currently processing. 250 SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> ModuleImportPath; 251 252 /// \brief Whether the last token we lexed was an '@'. 253 bool LastTokenWasAt; 254 255 /// \brief Whether the module import expects an identifier next. Otherwise, 256 /// it expects a '.' or ';'. 257 bool ModuleImportExpectsIdentifier; 258 259 /// \brief The source location of the currently-active 260 /// \#pragma clang arc_cf_code_audited begin. 261 SourceLocation PragmaARCCFCodeAuditedLoc; 262 263 /// \brief The source location of the currently-active 264 /// \#pragma clang assume_nonnull begin. 265 SourceLocation PragmaAssumeNonNullLoc; 266 267 /// \brief True if we hit the code-completion point. 268 bool CodeCompletionReached; 269 270 /// \brief The code completion token containing the information 271 /// on the stem that is to be code completed. 272 IdentifierInfo *CodeCompletionII; 273 274 /// \brief The directory that the main file should be considered to occupy, 275 /// if it does not correspond to a real file (as happens when building a 276 /// module). 277 const DirectoryEntry *MainFileDir; 278 279 /// \brief The number of bytes that we will initially skip when entering the 280 /// main file, along with a flag that indicates whether skipping this number 281 /// of bytes will place the lexer at the start of a line. 282 /// 283 /// This is used when loading a precompiled preamble. 284 std::pair<int, bool> SkipMainFilePreamble; 285 286 /// \brief The current top of the stack that we're lexing from if 287 /// not expanding a macro and we are lexing directly from source code. 288 /// 289 /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null. 290 std::unique_ptr<Lexer> CurLexer; 291 292 /// \brief The current top of stack that we're lexing from if 293 /// not expanding from a macro and we are lexing from a PTH cache. 294 /// 295 /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null. 296 std::unique_ptr<PTHLexer> CurPTHLexer; 297 298 /// \brief The current top of the stack what we're lexing from 299 /// if not expanding a macro. 300 /// 301 /// This is an alias for either CurLexer or CurPTHLexer. 302 PreprocessorLexer *CurPPLexer; 303 304 /// \brief Used to find the current FileEntry, if CurLexer is non-null 305 /// and if applicable. 306 /// 307 /// This allows us to implement \#include_next and find directory-specific 308 /// properties. 309 const DirectoryLookup *CurDirLookup; 310 311 /// \brief The current macro we are expanding, if we are expanding a macro. 312 /// 313 /// One of CurLexer and CurTokenLexer must be null. 314 std::unique_ptr<TokenLexer> CurTokenLexer; 315 316 /// \brief The kind of lexer we're currently working with. 317 enum CurLexerKind { 318 CLK_Lexer, 319 CLK_PTHLexer, 320 CLK_TokenLexer, 321 CLK_CachingLexer, 322 CLK_LexAfterModuleImport 323 } CurLexerKind; 324 325 /// \brief If the current lexer is for a submodule that is being built, this 326 /// is that submodule. 327 Module *CurSubmodule; 328 329 /// \brief Keeps track of the stack of files currently 330 /// \#included, and macros currently being expanded from, not counting 331 /// CurLexer/CurTokenLexer. 332 struct IncludeStackInfo { 333 enum CurLexerKind CurLexerKind; 334 Module *TheSubmodule; 335 std::unique_ptr<Lexer> TheLexer; 336 std::unique_ptr<PTHLexer> ThePTHLexer; 337 PreprocessorLexer *ThePPLexer; 338 std::unique_ptr<TokenLexer> TheTokenLexer; 339 const DirectoryLookup *TheDirLookup; 340 341 // The following constructors are completely useless copies of the default 342 // versions, only needed to pacify MSVC. 343 IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule, 344 std::unique_ptr<Lexer> &&TheLexer, 345 std::unique_ptr<PTHLexer> &&ThePTHLexer, 346 PreprocessorLexer *ThePPLexer, 347 std::unique_ptr<TokenLexer> &&TheTokenLexer, 348 const DirectoryLookup *TheDirLookup) 349 : CurLexerKind(std::move(CurLexerKind)), 350 TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)), 351 ThePTHLexer(std::move(ThePTHLexer)), 352 ThePPLexer(std::move(ThePPLexer)), 353 TheTokenLexer(std::move(TheTokenLexer)), 354 TheDirLookup(std::move(TheDirLookup)) {} 355 }; 356 std::vector<IncludeStackInfo> IncludeMacroStack; 357 358 /// \brief Actions invoked when some preprocessor activity is 359 /// encountered (e.g. a file is \#included, etc). 360 std::unique_ptr<PPCallbacks> Callbacks; 361 362 struct MacroExpandsInfo { 363 Token Tok; 364 MacroDefinition MD; 365 SourceRange Range; 366 MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range) 367 : Tok(Tok), MD(MD), Range(Range) { } 368 }; 369 SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks; 370 371 /// Information about a name that has been used to define a module macro. 372 struct ModuleMacroInfo { 373 ModuleMacroInfo(MacroDirective *MD) 374 : MD(MD), ActiveModuleMacrosGeneration(0), IsAmbiguous(false) {} 375 376 /// The most recent macro directive for this identifier. 377 MacroDirective *MD; 378 /// The active module macros for this identifier. 379 llvm::TinyPtrVector<ModuleMacro*> ActiveModuleMacros; 380 /// The generation number at which we last updated ActiveModuleMacros. 381 /// \see Preprocessor::VisibleModules. 382 unsigned ActiveModuleMacrosGeneration; 383 /// Whether this macro name is ambiguous. 384 bool IsAmbiguous; 385 /// The module macros that are overridden by this macro. 386 llvm::TinyPtrVector<ModuleMacro*> OverriddenMacros; 387 }; 388 389 /// The state of a macro for an identifier. 390 class MacroState { 391 mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State; 392 393 ModuleMacroInfo *getModuleInfo(Preprocessor &PP, 394 const IdentifierInfo *II) const { 395 if (II->isOutOfDate()) 396 PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II)); 397 // FIXME: Find a spare bit on IdentifierInfo and store a 398 // HasModuleMacros flag. 399 if (!II->hasMacroDefinition() || 400 (!PP.getLangOpts().Modules && 401 !PP.getLangOpts().ModulesLocalVisibility) || 402 !PP.CurSubmoduleState->VisibleModules.getGeneration()) 403 return nullptr; 404 405 auto *Info = State.dyn_cast<ModuleMacroInfo*>(); 406 if (!Info) { 407 Info = new (PP.getPreprocessorAllocator()) 408 ModuleMacroInfo(State.get<MacroDirective *>()); 409 State = Info; 410 } 411 412 if (PP.CurSubmoduleState->VisibleModules.getGeneration() != 413 Info->ActiveModuleMacrosGeneration) 414 PP.updateModuleMacroInfo(II, *Info); 415 return Info; 416 } 417 418 public: 419 MacroState() : MacroState(nullptr) {} 420 MacroState(MacroDirective *MD) : State(MD) {} 421 MacroState(MacroState &&O) noexcept : State(O.State) { 422 O.State = (MacroDirective *)nullptr; 423 } 424 MacroState &operator=(MacroState &&O) noexcept { 425 auto S = O.State; 426 O.State = (MacroDirective *)nullptr; 427 State = S; 428 return *this; 429 } 430 ~MacroState() { 431 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>()) 432 Info->~ModuleMacroInfo(); 433 } 434 435 MacroDirective *getLatest() const { 436 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>()) 437 return Info->MD; 438 return State.get<MacroDirective*>(); 439 } 440 void setLatest(MacroDirective *MD) { 441 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>()) 442 Info->MD = MD; 443 else 444 State = MD; 445 } 446 447 bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const { 448 auto *Info = getModuleInfo(PP, II); 449 return Info ? Info->IsAmbiguous : false; 450 } 451 ArrayRef<ModuleMacro *> 452 getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const { 453 if (auto *Info = getModuleInfo(PP, II)) 454 return Info->ActiveModuleMacros; 455 return None; 456 } 457 458 MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc, 459 SourceManager &SourceMgr) const { 460 // FIXME: Incorporate module macros into the result of this. 461 if (auto *Latest = getLatest()) 462 return Latest->findDirectiveAtLoc(Loc, SourceMgr); 463 return MacroDirective::DefInfo(); 464 } 465 466 void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) { 467 if (auto *Info = getModuleInfo(PP, II)) { 468 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(), 469 Info->ActiveModuleMacros.begin(), 470 Info->ActiveModuleMacros.end()); 471 Info->ActiveModuleMacros.clear(); 472 Info->IsAmbiguous = false; 473 } 474 } 475 ArrayRef<ModuleMacro*> getOverriddenMacros() const { 476 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>()) 477 return Info->OverriddenMacros; 478 return None; 479 } 480 void setOverriddenMacros(Preprocessor &PP, 481 ArrayRef<ModuleMacro *> Overrides) { 482 auto *Info = State.dyn_cast<ModuleMacroInfo*>(); 483 if (!Info) { 484 if (Overrides.empty()) 485 return; 486 Info = new (PP.getPreprocessorAllocator()) 487 ModuleMacroInfo(State.get<MacroDirective *>()); 488 State = Info; 489 } 490 Info->OverriddenMacros.clear(); 491 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(), 492 Overrides.begin(), Overrides.end()); 493 Info->ActiveModuleMacrosGeneration = 0; 494 } 495 }; 496 497 /// For each IdentifierInfo that was associated with a macro, we 498 /// keep a mapping to the history of all macro definitions and #undefs in 499 /// the reverse order (the latest one is in the head of the list). 500 /// 501 /// This mapping lives within the \p CurSubmoduleState. 502 typedef llvm::DenseMap<const IdentifierInfo *, MacroState> MacroMap; 503 504 friend class ASTReader; 505 506 struct SubmoduleState; 507 508 /// \brief Information about a submodule that we're currently building. 509 struct BuildingSubmoduleInfo { 510 BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, 511 SubmoduleState *OuterSubmoduleState, 512 unsigned OuterPendingModuleMacroNames) 513 : M(M), ImportLoc(ImportLoc), OuterSubmoduleState(OuterSubmoduleState), 514 OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {} 515 516 /// The module that we are building. 517 Module *M; 518 /// The location at which the module was included. 519 SourceLocation ImportLoc; 520 /// The previous SubmoduleState. 521 SubmoduleState *OuterSubmoduleState; 522 /// The number of pending module macro names when we started building this. 523 unsigned OuterPendingModuleMacroNames; 524 }; 525 SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack; 526 527 /// \brief Information about a submodule's preprocessor state. 528 struct SubmoduleState { 529 /// The macros for the submodule. 530 MacroMap Macros; 531 /// The set of modules that are visible within the submodule. 532 VisibleModuleSet VisibleModules; 533 // FIXME: CounterValue? 534 // FIXME: PragmaPushMacroInfo? 535 }; 536 std::map<Module*, SubmoduleState> Submodules; 537 538 /// The preprocessor state for preprocessing outside of any submodule. 539 SubmoduleState NullSubmoduleState; 540 541 /// The current submodule state. Will be \p NullSubmoduleState if we're not 542 /// in a submodule. 543 SubmoduleState *CurSubmoduleState; 544 545 /// The set of known macros exported from modules. 546 llvm::FoldingSet<ModuleMacro> ModuleMacros; 547 548 /// The names of potential module macros that we've not yet processed. 549 llvm::SmallVector<const IdentifierInfo*, 32> PendingModuleMacroNames; 550 551 /// The list of module macros, for each identifier, that are not overridden by 552 /// any other module macro. 553 llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro*>> 554 LeafModuleMacros; 555 556 /// \brief Macros that we want to warn because they are not used at the end 557 /// of the translation unit. 558 /// 559 /// We store just their SourceLocations instead of 560 /// something like MacroInfo*. The benefit of this is that when we are 561 /// deserializing from PCH, we don't need to deserialize identifier & macros 562 /// just so that we can report that they are unused, we just warn using 563 /// the SourceLocations of this set (that will be filled by the ASTReader). 564 /// We are using SmallPtrSet instead of a vector for faster removal. 565 typedef llvm::SmallPtrSet<SourceLocation, 32> WarnUnusedMacroLocsTy; 566 WarnUnusedMacroLocsTy WarnUnusedMacroLocs; 567 568 /// \brief A "freelist" of MacroArg objects that can be 569 /// reused for quick allocation. 570 MacroArgs *MacroArgCache; 571 friend class MacroArgs; 572 573 /// For each IdentifierInfo used in a \#pragma push_macro directive, 574 /// we keep a MacroInfo stack used to restore the previous macro value. 575 llvm::DenseMap<IdentifierInfo*, std::vector<MacroInfo*> > PragmaPushMacroInfo; 576 577 // Various statistics we track for performance analysis. 578 unsigned NumDirectives, NumDefined, NumUndefined, NumPragma; 579 unsigned NumIf, NumElse, NumEndif; 580 unsigned NumEnteredSourceFiles, MaxIncludeStackDepth; 581 unsigned NumMacroExpanded, NumFnMacroExpanded, NumBuiltinMacroExpanded; 582 unsigned NumFastMacroExpanded, NumTokenPaste, NumFastTokenPaste; 583 unsigned NumSkipped; 584 585 /// \brief The predefined macros that preprocessor should use from the 586 /// command line etc. 587 std::string Predefines; 588 589 /// \brief The file ID for the preprocessor predefines. 590 FileID PredefinesFileID; 591 592 /// \{ 593 /// \brief Cache of macro expanders to reduce malloc traffic. 594 enum { TokenLexerCacheSize = 8 }; 595 unsigned NumCachedTokenLexers; 596 std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize]; 597 /// \} 598 599 /// \brief Keeps macro expanded tokens for TokenLexers. 600 // 601 /// Works like a stack; a TokenLexer adds the macro expanded tokens that is 602 /// going to lex in the cache and when it finishes the tokens are removed 603 /// from the end of the cache. 604 SmallVector<Token, 16> MacroExpandedTokens; 605 std::vector<std::pair<TokenLexer *, size_t> > MacroExpandingLexersStack; 606 607 /// \brief A record of the macro definitions and expansions that 608 /// occurred during preprocessing. 609 /// 610 /// This is an optional side structure that can be enabled with 611 /// \c createPreprocessingRecord() prior to preprocessing. 612 PreprocessingRecord *Record; 613 614 /// Cached tokens state. 615 typedef SmallVector<Token, 1> CachedTokensTy; 616 617 /// \brief Cached tokens are stored here when we do backtracking or 618 /// lookahead. They are "lexed" by the CachingLex() method. 619 CachedTokensTy CachedTokens; 620 621 /// \brief The position of the cached token that CachingLex() should 622 /// "lex" next. 623 /// 624 /// If it points beyond the CachedTokens vector, it means that a normal 625 /// Lex() should be invoked. 626 CachedTokensTy::size_type CachedLexPos; 627 628 /// \brief Stack of backtrack positions, allowing nested backtracks. 629 /// 630 /// The EnableBacktrackAtThisPos() method pushes a position to 631 /// indicate where CachedLexPos should be set when the BackTrack() method is 632 /// invoked (at which point the last position is popped). 633 std::vector<CachedTokensTy::size_type> BacktrackPositions; 634 635 struct MacroInfoChain { 636 MacroInfo MI; 637 MacroInfoChain *Next; 638 }; 639 640 /// MacroInfos are managed as a chain for easy disposal. This is the head 641 /// of that list. 642 MacroInfoChain *MIChainHead; 643 644 struct DeserializedMacroInfoChain { 645 MacroInfo MI; 646 unsigned OwningModuleID; // MUST be immediately after the MacroInfo object 647 // so it can be accessed by MacroInfo::getOwningModuleID(). 648 DeserializedMacroInfoChain *Next; 649 }; 650 DeserializedMacroInfoChain *DeserialMIChainHead; 651 652 void updateOutOfDateIdentifier(IdentifierInfo &II) const; 653 654 public: 655 Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts, 656 DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, 657 MemoryBufferCache &PCMCache, 658 HeaderSearch &Headers, ModuleLoader &TheModuleLoader, 659 IdentifierInfoLookup *IILookup = nullptr, 660 bool OwnsHeaderSearch = false, 661 TranslationUnitKind TUKind = TU_Complete); 662 663 ~Preprocessor(); 664 665 /// \brief Initialize the preprocessor using information about the target. 666 /// 667 /// \param Target is owned by the caller and must remain valid for the 668 /// lifetime of the preprocessor. 669 /// \param AuxTarget is owned by the caller and must remain valid for 670 /// the lifetime of the preprocessor. 671 void Initialize(const TargetInfo &Target, 672 const TargetInfo *AuxTarget = nullptr); 673 674 /// \brief Initialize the preprocessor to parse a model file 675 /// 676 /// To parse model files the preprocessor of the original source is reused to 677 /// preserver the identifier table. However to avoid some duplicate 678 /// information in the preprocessor some cleanup is needed before it is used 679 /// to parse model files. This method does that cleanup. 680 void InitializeForModelFile(); 681 682 /// \brief Cleanup after model file parsing 683 void FinalizeForModelFile(); 684 685 /// \brief Retrieve the preprocessor options used to initialize this 686 /// preprocessor. 687 PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; } 688 689 DiagnosticsEngine &getDiagnostics() const { return *Diags; } 690 void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; } 691 692 const LangOptions &getLangOpts() const { return LangOpts; } 693 const TargetInfo &getTargetInfo() const { return *Target; } 694 const TargetInfo *getAuxTargetInfo() const { return AuxTarget; } 695 FileManager &getFileManager() const { return FileMgr; } 696 SourceManager &getSourceManager() const { return SourceMgr; } 697 MemoryBufferCache &getPCMCache() const { return PCMCache; } 698 HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; } 699 700 IdentifierTable &getIdentifierTable() { return Identifiers; } 701 const IdentifierTable &getIdentifierTable() const { return Identifiers; } 702 SelectorTable &getSelectorTable() { return Selectors; } 703 Builtin::Context &getBuiltinInfo() { return BuiltinInfo; } 704 llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; } 705 706 void setPTHManager(PTHManager* pm); 707 708 PTHManager *getPTHManager() { return PTH.get(); } 709 710 void setExternalSource(ExternalPreprocessorSource *Source) { 711 ExternalSource = Source; 712 } 713 714 ExternalPreprocessorSource *getExternalSource() const { 715 return ExternalSource; 716 } 717 718 /// \brief Retrieve the module loader associated with this preprocessor. 719 ModuleLoader &getModuleLoader() const { return TheModuleLoader; } 720 721 bool hadModuleLoaderFatalFailure() const { 722 return TheModuleLoader.HadFatalFailure; 723 } 724 725 /// \brief True if we are currently preprocessing a #if or #elif directive 726 bool isParsingIfOrElifDirective() const { 727 return ParsingIfOrElifDirective; 728 } 729 730 /// \brief Control whether the preprocessor retains comments in output. 731 void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) { 732 this->KeepComments = KeepComments | KeepMacroComments; 733 this->KeepMacroComments = KeepMacroComments; 734 } 735 736 bool getCommentRetentionState() const { return KeepComments; } 737 738 void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; } 739 bool getPragmasEnabled() const { return PragmasEnabled; } 740 741 void SetSuppressIncludeNotFoundError(bool Suppress) { 742 SuppressIncludeNotFoundError = Suppress; 743 } 744 745 bool GetSuppressIncludeNotFoundError() { 746 return SuppressIncludeNotFoundError; 747 } 748 749 /// Sets whether the preprocessor is responsible for producing output or if 750 /// it is producing tokens to be consumed by Parse and Sema. 751 void setPreprocessedOutput(bool IsPreprocessedOutput) { 752 PreprocessedOutput = IsPreprocessedOutput; 753 } 754 755 /// Returns true if the preprocessor is responsible for generating output, 756 /// false if it is producing tokens to be consumed by Parse and Sema. 757 bool isPreprocessedOutput() const { return PreprocessedOutput; } 758 759 /// \brief Return true if we are lexing directly from the specified lexer. 760 bool isCurrentLexer(const PreprocessorLexer *L) const { 761 return CurPPLexer == L; 762 } 763 764 /// \brief Return the current lexer being lexed from. 765 /// 766 /// Note that this ignores any potentially active macro expansions and _Pragma 767 /// expansions going on at the time. 768 PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; } 769 770 /// \brief Return the current file lexer being lexed from. 771 /// 772 /// Note that this ignores any potentially active macro expansions and _Pragma 773 /// expansions going on at the time. 774 PreprocessorLexer *getCurrentFileLexer() const; 775 776 /// \brief Return the submodule owning the file being lexed. 777 Module *getCurrentSubmodule() const { return CurSubmodule; } 778 779 /// \brief Returns the FileID for the preprocessor predefines. 780 FileID getPredefinesFileID() const { return PredefinesFileID; } 781 782 /// \{ 783 /// \brief Accessors for preprocessor callbacks. 784 /// 785 /// Note that this class takes ownership of any PPCallbacks object given to 786 /// it. 787 PPCallbacks *getPPCallbacks() const { return Callbacks.get(); } 788 void addPPCallbacks(std::unique_ptr<PPCallbacks> C) { 789 if (Callbacks) 790 C = llvm::make_unique<PPChainedCallbacks>(std::move(C), 791 std::move(Callbacks)); 792 Callbacks = std::move(C); 793 } 794 /// \} 795 796 bool isMacroDefined(StringRef Id) { 797 return isMacroDefined(&Identifiers.get(Id)); 798 } 799 bool isMacroDefined(const IdentifierInfo *II) { 800 return II->hasMacroDefinition() && 801 (!getLangOpts().Modules || (bool)getMacroDefinition(II)); 802 } 803 804 /// \brief Determine whether II is defined as a macro within the module M, 805 /// if that is a module that we've already preprocessed. Does not check for 806 /// macros imported into M. 807 bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M) { 808 if (!II->hasMacroDefinition()) 809 return false; 810 auto I = Submodules.find(M); 811 if (I == Submodules.end()) 812 return false; 813 auto J = I->second.Macros.find(II); 814 if (J == I->second.Macros.end()) 815 return false; 816 auto *MD = J->second.getLatest(); 817 return MD && MD->isDefined(); 818 } 819 820 MacroDefinition getMacroDefinition(const IdentifierInfo *II) { 821 if (!II->hasMacroDefinition()) 822 return MacroDefinition(); 823 824 MacroState &S = CurSubmoduleState->Macros[II]; 825 auto *MD = S.getLatest(); 826 while (MD && isa<VisibilityMacroDirective>(MD)) 827 MD = MD->getPrevious(); 828 return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD), 829 S.getActiveModuleMacros(*this, II), 830 S.isAmbiguous(*this, II)); 831 } 832 833 MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, 834 SourceLocation Loc) { 835 if (!II->hadMacroDefinition()) 836 return MacroDefinition(); 837 838 MacroState &S = CurSubmoduleState->Macros[II]; 839 MacroDirective::DefInfo DI; 840 if (auto *MD = S.getLatest()) 841 DI = MD->findDirectiveAtLoc(Loc, getSourceManager()); 842 // FIXME: Compute the set of active module macros at the specified location. 843 return MacroDefinition(DI.getDirective(), 844 S.getActiveModuleMacros(*this, II), 845 S.isAmbiguous(*this, II)); 846 } 847 848 /// \brief Given an identifier, return its latest non-imported MacroDirective 849 /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd. 850 MacroDirective *getLocalMacroDirective(const IdentifierInfo *II) const { 851 if (!II->hasMacroDefinition()) 852 return nullptr; 853 854 auto *MD = getLocalMacroDirectiveHistory(II); 855 if (!MD || MD->getDefinition().isUndefined()) 856 return nullptr; 857 858 return MD; 859 } 860 861 const MacroInfo *getMacroInfo(const IdentifierInfo *II) const { 862 return const_cast<Preprocessor*>(this)->getMacroInfo(II); 863 } 864 865 MacroInfo *getMacroInfo(const IdentifierInfo *II) { 866 if (!II->hasMacroDefinition()) 867 return nullptr; 868 if (auto MD = getMacroDefinition(II)) 869 return MD.getMacroInfo(); 870 return nullptr; 871 } 872 873 /// \brief Given an identifier, return the latest non-imported macro 874 /// directive for that identifier. 875 /// 876 /// One can iterate over all previous macro directives from the most recent 877 /// one. 878 MacroDirective *getLocalMacroDirectiveHistory(const IdentifierInfo *II) const; 879 880 /// \brief Add a directive to the macro directive history for this identifier. 881 void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD); 882 DefMacroDirective *appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, 883 SourceLocation Loc) { 884 DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc); 885 appendMacroDirective(II, MD); 886 return MD; 887 } 888 DefMacroDirective *appendDefMacroDirective(IdentifierInfo *II, 889 MacroInfo *MI) { 890 return appendDefMacroDirective(II, MI, MI->getDefinitionLoc()); 891 } 892 /// \brief Set a MacroDirective that was loaded from a PCH file. 893 void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED, 894 MacroDirective *MD); 895 896 /// \brief Register an exported macro for a module and identifier. 897 ModuleMacro *addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, 898 ArrayRef<ModuleMacro *> Overrides, bool &IsNew); 899 ModuleMacro *getModuleMacro(Module *Mod, IdentifierInfo *II); 900 901 /// \brief Get the list of leaf (non-overridden) module macros for a name. 902 ArrayRef<ModuleMacro*> getLeafModuleMacros(const IdentifierInfo *II) const { 903 if (II->isOutOfDate()) 904 updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II)); 905 auto I = LeafModuleMacros.find(II); 906 if (I != LeafModuleMacros.end()) 907 return I->second; 908 return None; 909 } 910 911 /// \{ 912 /// Iterators for the macro history table. Currently defined macros have 913 /// IdentifierInfo::hasMacroDefinition() set and an empty 914 /// MacroInfo::getUndefLoc() at the head of the list. 915 typedef MacroMap::const_iterator macro_iterator; 916 macro_iterator macro_begin(bool IncludeExternalMacros = true) const; 917 macro_iterator macro_end(bool IncludeExternalMacros = true) const; 918 llvm::iterator_range<macro_iterator> 919 macros(bool IncludeExternalMacros = true) const { 920 return llvm::make_range(macro_begin(IncludeExternalMacros), 921 macro_end(IncludeExternalMacros)); 922 } 923 /// \} 924 925 /// \brief Return the name of the macro defined before \p Loc that has 926 /// spelling \p Tokens. If there are multiple macros with same spelling, 927 /// return the last one defined. 928 StringRef getLastMacroWithSpelling(SourceLocation Loc, 929 ArrayRef<TokenValue> Tokens) const; 930 931 const std::string &getPredefines() const { return Predefines; } 932 /// \brief Set the predefines for this Preprocessor. 933 /// 934 /// These predefines are automatically injected when parsing the main file. 935 void setPredefines(const char *P) { Predefines = P; } 936 void setPredefines(StringRef P) { Predefines = P; } 937 938 /// Return information about the specified preprocessor 939 /// identifier token. 940 IdentifierInfo *getIdentifierInfo(StringRef Name) const { 941 return &Identifiers.get(Name); 942 } 943 944 /// \brief Add the specified pragma handler to this preprocessor. 945 /// 946 /// If \p Namespace is non-null, then it is a token required to exist on the 947 /// pragma line before the pragma string starts, e.g. "STDC" or "GCC". 948 void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler); 949 void AddPragmaHandler(PragmaHandler *Handler) { 950 AddPragmaHandler(StringRef(), Handler); 951 } 952 953 /// \brief Remove the specific pragma handler from this preprocessor. 954 /// 955 /// If \p Namespace is non-null, then it should be the namespace that 956 /// \p Handler was added to. It is an error to remove a handler that 957 /// has not been registered. 958 void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler); 959 void RemovePragmaHandler(PragmaHandler *Handler) { 960 RemovePragmaHandler(StringRef(), Handler); 961 } 962 963 /// Install empty handlers for all pragmas (making them ignored). 964 void IgnorePragmas(); 965 966 /// \brief Add the specified comment handler to the preprocessor. 967 void addCommentHandler(CommentHandler *Handler); 968 969 /// \brief Remove the specified comment handler. 970 /// 971 /// It is an error to remove a handler that has not been registered. 972 void removeCommentHandler(CommentHandler *Handler); 973 974 /// \brief Set the code completion handler to the given object. 975 void setCodeCompletionHandler(CodeCompletionHandler &Handler) { 976 CodeComplete = &Handler; 977 } 978 979 /// \brief Retrieve the current code-completion handler. 980 CodeCompletionHandler *getCodeCompletionHandler() const { 981 return CodeComplete; 982 } 983 984 /// \brief Clear out the code completion handler. 985 void clearCodeCompletionHandler() { 986 CodeComplete = nullptr; 987 } 988 989 /// \brief Hook used by the lexer to invoke the "natural language" code 990 /// completion point. 991 void CodeCompleteNaturalLanguage(); 992 993 /// \brief Set the code completion token for filtering purposes. 994 void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter) { 995 CodeCompletionII = Filter; 996 } 997 998 /// \brief Get the code completion token for filtering purposes. 999 StringRef getCodeCompletionFilter() { 1000 if (CodeCompletionII) 1001 return CodeCompletionII->getName(); 1002 return {}; 1003 } 1004 1005 /// \brief Retrieve the preprocessing record, or NULL if there is no 1006 /// preprocessing record. 1007 PreprocessingRecord *getPreprocessingRecord() const { return Record; } 1008 1009 /// \brief Create a new preprocessing record, which will keep track of 1010 /// all macro expansions, macro definitions, etc. 1011 void createPreprocessingRecord(); 1012 1013 /// \brief Enter the specified FileID as the main source file, 1014 /// which implicitly adds the builtin defines etc. 1015 void EnterMainSourceFile(); 1016 1017 /// \brief Inform the preprocessor callbacks that processing is complete. 1018 void EndSourceFile(); 1019 1020 /// \brief Add a source file to the top of the include stack and 1021 /// start lexing tokens from it instead of the current buffer. 1022 /// 1023 /// Emits a diagnostic, doesn't enter the file, and returns true on error. 1024 bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir, 1025 SourceLocation Loc); 1026 1027 /// \brief Add a Macro to the top of the include stack and start lexing 1028 /// tokens from it instead of the current buffer. 1029 /// 1030 /// \param Args specifies the tokens input to a function-like macro. 1031 /// \param ILEnd specifies the location of the ')' for a function-like macro 1032 /// or the identifier for an object-like macro. 1033 void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro, 1034 MacroArgs *Args); 1035 1036 /// \brief Add a "macro" context to the top of the include stack, 1037 /// which will cause the lexer to start returning the specified tokens. 1038 /// 1039 /// If \p DisableMacroExpansion is true, tokens lexed from the token stream 1040 /// will not be subject to further macro expansion. Otherwise, these tokens 1041 /// will be re-macro-expanded when/if expansion is enabled. 1042 /// 1043 /// If \p OwnsTokens is false, this method assumes that the specified stream 1044 /// of tokens has a permanent owner somewhere, so they do not need to be 1045 /// copied. If it is true, it assumes the array of tokens is allocated with 1046 /// \c new[] and the Preprocessor will delete[] it. 1047 private: 1048 void EnterTokenStream(const Token *Toks, unsigned NumToks, 1049 bool DisableMacroExpansion, bool OwnsTokens); 1050 1051 public: 1052 void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks, 1053 bool DisableMacroExpansion) { 1054 EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true); 1055 } 1056 void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion) { 1057 EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false); 1058 } 1059 1060 /// \brief Pop the current lexer/macro exp off the top of the lexer stack. 1061 /// 1062 /// This should only be used in situations where the current state of the 1063 /// top-of-stack lexer is known. 1064 void RemoveTopOfLexerStack(); 1065 1066 /// From the point that this method is called, and until 1067 /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor 1068 /// keeps track of the lexed tokens so that a subsequent Backtrack() call will 1069 /// make the Preprocessor re-lex the same tokens. 1070 /// 1071 /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can 1072 /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will 1073 /// be combined with the EnableBacktrackAtThisPos calls in reverse order. 1074 /// 1075 /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack 1076 /// at some point after EnableBacktrackAtThisPos. If you don't, caching of 1077 /// tokens will continue indefinitely. 1078 /// 1079 void EnableBacktrackAtThisPos(); 1080 1081 /// \brief Disable the last EnableBacktrackAtThisPos call. 1082 void CommitBacktrackedTokens(); 1083 1084 struct CachedTokensRange { 1085 CachedTokensTy::size_type Begin, End; 1086 }; 1087 1088 private: 1089 /// \brief A range of cached tokens that should be erased after lexing 1090 /// when backtracking requires the erasure of such cached tokens. 1091 Optional<CachedTokensRange> CachedTokenRangeToErase; 1092 1093 public: 1094 /// \brief Returns the range of cached tokens that were lexed since 1095 /// EnableBacktrackAtThisPos() was previously called. 1096 CachedTokensRange LastCachedTokenRange(); 1097 1098 /// \brief Erase the range of cached tokens that were lexed since 1099 /// EnableBacktrackAtThisPos() was previously called. 1100 void EraseCachedTokens(CachedTokensRange TokenRange); 1101 1102 /// \brief Make Preprocessor re-lex the tokens that were lexed since 1103 /// EnableBacktrackAtThisPos() was previously called. 1104 void Backtrack(); 1105 1106 /// \brief True if EnableBacktrackAtThisPos() was called and 1107 /// caching of tokens is on. 1108 bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); } 1109 1110 /// \brief Lex the next token for this preprocessor. 1111 void Lex(Token &Result); 1112 1113 void LexAfterModuleImport(Token &Result); 1114 1115 void makeModuleVisible(Module *M, SourceLocation Loc); 1116 1117 SourceLocation getModuleImportLoc(Module *M) const { 1118 return CurSubmoduleState->VisibleModules.getImportLoc(M); 1119 } 1120 1121 /// \brief Lex a string literal, which may be the concatenation of multiple 1122 /// string literals and may even come from macro expansion. 1123 /// \returns true on success, false if a error diagnostic has been generated. 1124 bool LexStringLiteral(Token &Result, std::string &String, 1125 const char *DiagnosticTag, bool AllowMacroExpansion) { 1126 if (AllowMacroExpansion) 1127 Lex(Result); 1128 else 1129 LexUnexpandedToken(Result); 1130 return FinishLexStringLiteral(Result, String, DiagnosticTag, 1131 AllowMacroExpansion); 1132 } 1133 1134 /// \brief Complete the lexing of a string literal where the first token has 1135 /// already been lexed (see LexStringLiteral). 1136 bool FinishLexStringLiteral(Token &Result, std::string &String, 1137 const char *DiagnosticTag, 1138 bool AllowMacroExpansion); 1139 1140 /// \brief Lex a token. If it's a comment, keep lexing until we get 1141 /// something not a comment. 1142 /// 1143 /// This is useful in -E -C mode where comments would foul up preprocessor 1144 /// directive handling. 1145 void LexNonComment(Token &Result) { 1146 do 1147 Lex(Result); 1148 while (Result.getKind() == tok::comment); 1149 } 1150 1151 /// \brief Just like Lex, but disables macro expansion of identifier tokens. 1152 void LexUnexpandedToken(Token &Result) { 1153 // Disable macro expansion. 1154 bool OldVal = DisableMacroExpansion; 1155 DisableMacroExpansion = true; 1156 // Lex the token. 1157 Lex(Result); 1158 1159 // Reenable it. 1160 DisableMacroExpansion = OldVal; 1161 } 1162 1163 /// \brief Like LexNonComment, but this disables macro expansion of 1164 /// identifier tokens. 1165 void LexUnexpandedNonComment(Token &Result) { 1166 do 1167 LexUnexpandedToken(Result); 1168 while (Result.getKind() == tok::comment); 1169 } 1170 1171 /// \brief Parses a simple integer literal to get its numeric value. Floating 1172 /// point literals and user defined literals are rejected. Used primarily to 1173 /// handle pragmas that accept integer arguments. 1174 bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value); 1175 1176 /// Disables macro expansion everywhere except for preprocessor directives. 1177 void SetMacroExpansionOnlyInDirectives() { 1178 DisableMacroExpansion = true; 1179 MacroExpansionInDirectivesOverride = true; 1180 } 1181 1182 /// \brief Peeks ahead N tokens and returns that token without consuming any 1183 /// tokens. 1184 /// 1185 /// LookAhead(0) returns the next token that would be returned by Lex(), 1186 /// LookAhead(1) returns the token after it, etc. This returns normal 1187 /// tokens after phase 5. As such, it is equivalent to using 1188 /// 'Lex', not 'LexUnexpandedToken'. 1189 const Token &LookAhead(unsigned N) { 1190 if (CachedLexPos + N < CachedTokens.size()) 1191 return CachedTokens[CachedLexPos+N]; 1192 else 1193 return PeekAhead(N+1); 1194 } 1195 1196 /// \brief When backtracking is enabled and tokens are cached, 1197 /// this allows to revert a specific number of tokens. 1198 /// 1199 /// Note that the number of tokens being reverted should be up to the last 1200 /// backtrack position, not more. 1201 void RevertCachedTokens(unsigned N) { 1202 assert(isBacktrackEnabled() && 1203 "Should only be called when tokens are cached for backtracking"); 1204 assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back()) 1205 && "Should revert tokens up to the last backtrack position, not more"); 1206 assert(signed(CachedLexPos) - signed(N) >= 0 && 1207 "Corrupted backtrack positions ?"); 1208 CachedLexPos -= N; 1209 } 1210 1211 /// \brief Enters a token in the token stream to be lexed next. 1212 /// 1213 /// If BackTrack() is called afterwards, the token will remain at the 1214 /// insertion point. 1215 void EnterToken(const Token &Tok) { 1216 EnterCachingLexMode(); 1217 CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok); 1218 } 1219 1220 /// We notify the Preprocessor that if it is caching tokens (because 1221 /// backtrack is enabled) it should replace the most recent cached tokens 1222 /// with the given annotation token. This function has no effect if 1223 /// backtracking is not enabled. 1224 /// 1225 /// Note that the use of this function is just for optimization, so that the 1226 /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is 1227 /// invoked. 1228 void AnnotateCachedTokens(const Token &Tok) { 1229 assert(Tok.isAnnotation() && "Expected annotation token"); 1230 if (CachedLexPos != 0 && isBacktrackEnabled()) 1231 AnnotatePreviousCachedTokens(Tok); 1232 } 1233 1234 /// Get the location of the last cached token, suitable for setting the end 1235 /// location of an annotation token. 1236 SourceLocation getLastCachedTokenLocation() const { 1237 assert(CachedLexPos != 0); 1238 return CachedTokens[CachedLexPos-1].getLastLoc(); 1239 } 1240 1241 /// \brief Whether \p Tok is the most recent token (`CachedLexPos - 1`) in 1242 /// CachedTokens. 1243 bool IsPreviousCachedToken(const Token &Tok) const; 1244 1245 /// \brief Replace token in `CachedLexPos - 1` in CachedTokens by the tokens 1246 /// in \p NewToks. 1247 /// 1248 /// Useful when a token needs to be split in smaller ones and CachedTokens 1249 /// most recent token must to be updated to reflect that. 1250 void ReplacePreviousCachedToken(ArrayRef<Token> NewToks); 1251 1252 /// \brief Replace the last token with an annotation token. 1253 /// 1254 /// Like AnnotateCachedTokens(), this routine replaces an 1255 /// already-parsed (and resolved) token with an annotation 1256 /// token. However, this routine only replaces the last token with 1257 /// the annotation token; it does not affect any other cached 1258 /// tokens. This function has no effect if backtracking is not 1259 /// enabled. 1260 void ReplaceLastTokenWithAnnotation(const Token &Tok) { 1261 assert(Tok.isAnnotation() && "Expected annotation token"); 1262 if (CachedLexPos != 0 && isBacktrackEnabled()) 1263 CachedTokens[CachedLexPos-1] = Tok; 1264 } 1265 1266 /// Update the current token to represent the provided 1267 /// identifier, in order to cache an action performed by typo correction. 1268 void TypoCorrectToken(const Token &Tok) { 1269 assert(Tok.getIdentifierInfo() && "Expected identifier token"); 1270 if (CachedLexPos != 0 && isBacktrackEnabled()) 1271 CachedTokens[CachedLexPos-1] = Tok; 1272 } 1273 1274 /// \brief Recompute the current lexer kind based on the CurLexer/CurPTHLexer/ 1275 /// CurTokenLexer pointers. 1276 void recomputeCurLexerKind(); 1277 1278 /// \brief Returns true if incremental processing is enabled 1279 bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; } 1280 1281 /// \brief Enables the incremental processing 1282 void enableIncrementalProcessing(bool value = true) { 1283 IncrementalProcessing = value; 1284 } 1285 1286 /// \brief Specify the point at which code-completion will be performed. 1287 /// 1288 /// \param File the file in which code completion should occur. If 1289 /// this file is included multiple times, code-completion will 1290 /// perform completion the first time it is included. If NULL, this 1291 /// function clears out the code-completion point. 1292 /// 1293 /// \param Line the line at which code completion should occur 1294 /// (1-based). 1295 /// 1296 /// \param Column the column at which code completion should occur 1297 /// (1-based). 1298 /// 1299 /// \returns true if an error occurred, false otherwise. 1300 bool SetCodeCompletionPoint(const FileEntry *File, 1301 unsigned Line, unsigned Column); 1302 1303 /// \brief Determine if we are performing code completion. 1304 bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; } 1305 1306 /// \brief Returns the location of the code-completion point. 1307 /// 1308 /// Returns an invalid location if code-completion is not enabled or the file 1309 /// containing the code-completion point has not been lexed yet. 1310 SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; } 1311 1312 /// \brief Returns the start location of the file of code-completion point. 1313 /// 1314 /// Returns an invalid location if code-completion is not enabled or the file 1315 /// containing the code-completion point has not been lexed yet. 1316 SourceLocation getCodeCompletionFileLoc() const { 1317 return CodeCompletionFileLoc; 1318 } 1319 1320 /// \brief Returns true if code-completion is enabled and we have hit the 1321 /// code-completion point. 1322 bool isCodeCompletionReached() const { return CodeCompletionReached; } 1323 1324 /// \brief Note that we hit the code-completion point. 1325 void setCodeCompletionReached() { 1326 assert(isCodeCompletionEnabled() && "Code-completion not enabled!"); 1327 CodeCompletionReached = true; 1328 // Silence any diagnostics that occur after we hit the code-completion. 1329 getDiagnostics().setSuppressAllDiagnostics(true); 1330 } 1331 1332 /// \brief The location of the currently-active \#pragma clang 1333 /// arc_cf_code_audited begin. 1334 /// 1335 /// Returns an invalid location if there is no such pragma active. 1336 SourceLocation getPragmaARCCFCodeAuditedLoc() const { 1337 return PragmaARCCFCodeAuditedLoc; 1338 } 1339 1340 /// \brief Set the location of the currently-active \#pragma clang 1341 /// arc_cf_code_audited begin. An invalid location ends the pragma. 1342 void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc) { 1343 PragmaARCCFCodeAuditedLoc = Loc; 1344 } 1345 1346 /// \brief The location of the currently-active \#pragma clang 1347 /// assume_nonnull begin. 1348 /// 1349 /// Returns an invalid location if there is no such pragma active. 1350 SourceLocation getPragmaAssumeNonNullLoc() const { 1351 return PragmaAssumeNonNullLoc; 1352 } 1353 1354 /// \brief Set the location of the currently-active \#pragma clang 1355 /// assume_nonnull begin. An invalid location ends the pragma. 1356 void setPragmaAssumeNonNullLoc(SourceLocation Loc) { 1357 PragmaAssumeNonNullLoc = Loc; 1358 } 1359 1360 /// \brief Set the directory in which the main file should be considered 1361 /// to have been found, if it is not a real file. 1362 void setMainFileDir(const DirectoryEntry *Dir) { 1363 MainFileDir = Dir; 1364 } 1365 1366 /// \brief Instruct the preprocessor to skip part of the main source file. 1367 /// 1368 /// \param Bytes The number of bytes in the preamble to skip. 1369 /// 1370 /// \param StartOfLine Whether skipping these bytes puts the lexer at the 1371 /// start of a line. 1372 void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) { 1373 SkipMainFilePreamble.first = Bytes; 1374 SkipMainFilePreamble.second = StartOfLine; 1375 } 1376 1377 /// Forwarding function for diagnostics. This emits a diagnostic at 1378 /// the specified Token's location, translating the token's start 1379 /// position in the current buffer into a SourcePosition object for rendering. 1380 DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const { 1381 return Diags->Report(Loc, DiagID); 1382 } 1383 1384 DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const { 1385 return Diags->Report(Tok.getLocation(), DiagID); 1386 } 1387 1388 /// Return the 'spelling' of the token at the given 1389 /// location; does not go up to the spelling location or down to the 1390 /// expansion location. 1391 /// 1392 /// \param buffer A buffer which will be used only if the token requires 1393 /// "cleaning", e.g. if it contains trigraphs or escaped newlines 1394 /// \param invalid If non-null, will be set \c true if an error occurs. 1395 StringRef getSpelling(SourceLocation loc, 1396 SmallVectorImpl<char> &buffer, 1397 bool *invalid = nullptr) const { 1398 return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid); 1399 } 1400 1401 /// \brief Return the 'spelling' of the Tok token. 1402 /// 1403 /// The spelling of a token is the characters used to represent the token in 1404 /// the source file after trigraph expansion and escaped-newline folding. In 1405 /// particular, this wants to get the true, uncanonicalized, spelling of 1406 /// things like digraphs, UCNs, etc. 1407 /// 1408 /// \param Invalid If non-null, will be set \c true if an error occurs. 1409 std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const { 1410 return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid); 1411 } 1412 1413 /// \brief Get the spelling of a token into a preallocated buffer, instead 1414 /// of as an std::string. 1415 /// 1416 /// The caller is required to allocate enough space for the token, which is 1417 /// guaranteed to be at least Tok.getLength() bytes long. The length of the 1418 /// actual result is returned. 1419 /// 1420 /// Note that this method may do two possible things: it may either fill in 1421 /// the buffer specified with characters, or it may *change the input pointer* 1422 /// to point to a constant buffer with the data already in it (avoiding a 1423 /// copy). The caller is not allowed to modify the returned buffer pointer 1424 /// if an internal buffer is returned. 1425 unsigned getSpelling(const Token &Tok, const char *&Buffer, 1426 bool *Invalid = nullptr) const { 1427 return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid); 1428 } 1429 1430 /// \brief Get the spelling of a token into a SmallVector. 1431 /// 1432 /// Note that the returned StringRef may not point to the 1433 /// supplied buffer if a copy can be avoided. 1434 StringRef getSpelling(const Token &Tok, 1435 SmallVectorImpl<char> &Buffer, 1436 bool *Invalid = nullptr) const; 1437 1438 /// \brief Relex the token at the specified location. 1439 /// \returns true if there was a failure, false on success. 1440 bool getRawToken(SourceLocation Loc, Token &Result, 1441 bool IgnoreWhiteSpace = false) { 1442 return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace); 1443 } 1444 1445 /// \brief Given a Token \p Tok that is a numeric constant with length 1, 1446 /// return the character. 1447 char 1448 getSpellingOfSingleCharacterNumericConstant(const Token &Tok, 1449 bool *Invalid = nullptr) const { 1450 assert(Tok.is(tok::numeric_constant) && 1451 Tok.getLength() == 1 && "Called on unsupported token"); 1452 assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1"); 1453 1454 // If the token is carrying a literal data pointer, just use it. 1455 if (const char *D = Tok.getLiteralData()) 1456 return *D; 1457 1458 // Otherwise, fall back on getCharacterData, which is slower, but always 1459 // works. 1460 return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid); 1461 } 1462 1463 /// \brief Retrieve the name of the immediate macro expansion. 1464 /// 1465 /// This routine starts from a source location, and finds the name of the 1466 /// macro responsible for its immediate expansion. It looks through any 1467 /// intervening macro argument expansions to compute this. It returns a 1468 /// StringRef that refers to the SourceManager-owned buffer of the source 1469 /// where that macro name is spelled. Thus, the result shouldn't out-live 1470 /// the SourceManager. 1471 StringRef getImmediateMacroName(SourceLocation Loc) { 1472 return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts()); 1473 } 1474 1475 /// \brief Plop the specified string into a scratch buffer and set the 1476 /// specified token's location and length to it. 1477 /// 1478 /// If specified, the source location provides a location of the expansion 1479 /// point of the token. 1480 void CreateString(StringRef Str, Token &Tok, 1481 SourceLocation ExpansionLocStart = SourceLocation(), 1482 SourceLocation ExpansionLocEnd = SourceLocation()); 1483 1484 /// \brief Computes the source location just past the end of the 1485 /// token at this source location. 1486 /// 1487 /// This routine can be used to produce a source location that 1488 /// points just past the end of the token referenced by \p Loc, and 1489 /// is generally used when a diagnostic needs to point just after a 1490 /// token where it expected something different that it received. If 1491 /// the returned source location would not be meaningful (e.g., if 1492 /// it points into a macro), this routine returns an invalid 1493 /// source location. 1494 /// 1495 /// \param Offset an offset from the end of the token, where the source 1496 /// location should refer to. The default offset (0) produces a source 1497 /// location pointing just past the end of the token; an offset of 1 produces 1498 /// a source location pointing to the last character in the token, etc. 1499 SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset = 0) { 1500 return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts); 1501 } 1502 1503 /// \brief Returns true if the given MacroID location points at the first 1504 /// token of the macro expansion. 1505 /// 1506 /// \param MacroBegin If non-null and function returns true, it is set to 1507 /// begin location of the macro. 1508 bool isAtStartOfMacroExpansion(SourceLocation loc, 1509 SourceLocation *MacroBegin = nullptr) const { 1510 return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts, 1511 MacroBegin); 1512 } 1513 1514 /// \brief Returns true if the given MacroID location points at the last 1515 /// token of the macro expansion. 1516 /// 1517 /// \param MacroEnd If non-null and function returns true, it is set to 1518 /// end location of the macro. 1519 bool isAtEndOfMacroExpansion(SourceLocation loc, 1520 SourceLocation *MacroEnd = nullptr) const { 1521 return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd); 1522 } 1523 1524 /// \brief Print the token to stderr, used for debugging. 1525 void DumpToken(const Token &Tok, bool DumpFlags = false) const; 1526 void DumpLocation(SourceLocation Loc) const; 1527 void DumpMacro(const MacroInfo &MI) const; 1528 void dumpMacroInfo(const IdentifierInfo *II); 1529 1530 /// \brief Given a location that specifies the start of a 1531 /// token, return a new location that specifies a character within the token. 1532 SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, 1533 unsigned Char) const { 1534 return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts); 1535 } 1536 1537 /// \brief Increment the counters for the number of token paste operations 1538 /// performed. 1539 /// 1540 /// If fast was specified, this is a 'fast paste' case we handled. 1541 void IncrementPasteCounter(bool isFast) { 1542 if (isFast) 1543 ++NumFastTokenPaste; 1544 else 1545 ++NumTokenPaste; 1546 } 1547 1548 void PrintStats(); 1549 1550 size_t getTotalMemory() const; 1551 1552 /// When the macro expander pastes together a comment (/##/) in Microsoft 1553 /// mode, this method handles updating the current state, returning the 1554 /// token on the next source line. 1555 void HandleMicrosoftCommentPaste(Token &Tok); 1556 1557 //===--------------------------------------------------------------------===// 1558 // Preprocessor callback methods. These are invoked by a lexer as various 1559 // directives and events are found. 1560 1561 /// Given a tok::raw_identifier token, look up the 1562 /// identifier information for the token and install it into the token, 1563 /// updating the token kind accordingly. 1564 IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const; 1565 1566 private: 1567 llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons; 1568 1569 public: 1570 1571 /// \brief Specifies the reason for poisoning an identifier. 1572 /// 1573 /// If that identifier is accessed while poisoned, then this reason will be 1574 /// used instead of the default "poisoned" diagnostic. 1575 void SetPoisonReason(IdentifierInfo *II, unsigned DiagID); 1576 1577 /// \brief Display reason for poisoned identifier. 1578 void HandlePoisonedIdentifier(Token & Tok); 1579 1580 void MaybeHandlePoisonedIdentifier(Token & Identifier) { 1581 if(IdentifierInfo * II = Identifier.getIdentifierInfo()) { 1582 if(II->isPoisoned()) { 1583 HandlePoisonedIdentifier(Identifier); 1584 } 1585 } 1586 } 1587 1588 private: 1589 /// Identifiers used for SEH handling in Borland. These are only 1590 /// allowed in particular circumstances 1591 // __except block 1592 IdentifierInfo *Ident__exception_code, 1593 *Ident___exception_code, 1594 *Ident_GetExceptionCode; 1595 // __except filter expression 1596 IdentifierInfo *Ident__exception_info, 1597 *Ident___exception_info, 1598 *Ident_GetExceptionInfo; 1599 // __finally 1600 IdentifierInfo *Ident__abnormal_termination, 1601 *Ident___abnormal_termination, 1602 *Ident_AbnormalTermination; 1603 1604 const char *getCurLexerEndPos(); 1605 1606 public: 1607 void PoisonSEHIdentifiers(bool Poison = true); // Borland 1608 1609 /// \brief Callback invoked when the lexer reads an identifier and has 1610 /// filled in the tokens IdentifierInfo member. 1611 /// 1612 /// This callback potentially macro expands it or turns it into a named 1613 /// token (like 'for'). 1614 /// 1615 /// \returns true if we actually computed a token, false if we need to 1616 /// lex again. 1617 bool HandleIdentifier(Token &Identifier); 1618 1619 1620 /// \brief Callback invoked when the lexer hits the end of the current file. 1621 /// 1622 /// This either returns the EOF token and returns true, or 1623 /// pops a level off the include stack and returns false, at which point the 1624 /// client should call lex again. 1625 bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false); 1626 1627 /// \brief Callback invoked when the current TokenLexer hits the end of its 1628 /// token stream. 1629 bool HandleEndOfTokenLexer(Token &Result); 1630 1631 /// \brief Callback invoked when the lexer sees a # token at the start of a 1632 /// line. 1633 /// 1634 /// This consumes the directive, modifies the lexer/preprocessor state, and 1635 /// advances the lexer(s) so that the next token read is the correct one. 1636 void HandleDirective(Token &Result); 1637 1638 /// \brief Ensure that the next token is a tok::eod token. 1639 /// 1640 /// If not, emit a diagnostic and consume up until the eod. 1641 /// If \p EnableMacros is true, then we consider macros that expand to zero 1642 /// tokens as being ok. 1643 void CheckEndOfDirective(const char *Directive, bool EnableMacros = false); 1644 1645 /// \brief Read and discard all tokens remaining on the current line until 1646 /// the tok::eod token is found. 1647 void DiscardUntilEndOfDirective(); 1648 1649 /// \brief Returns true if the preprocessor has seen a use of 1650 /// __DATE__ or __TIME__ in the file so far. 1651 bool SawDateOrTime() const { 1652 return DATELoc != SourceLocation() || TIMELoc != SourceLocation(); 1653 } 1654 unsigned getCounterValue() const { return CounterValue; } 1655 void setCounterValue(unsigned V) { CounterValue = V; } 1656 1657 /// \brief Retrieves the module that we're currently building, if any. 1658 Module *getCurrentModule(); 1659 1660 /// \brief Allocate a new MacroInfo object with the provided SourceLocation. 1661 MacroInfo *AllocateMacroInfo(SourceLocation L); 1662 1663 /// \brief Allocate a new MacroInfo object loaded from an AST file. 1664 MacroInfo *AllocateDeserializedMacroInfo(SourceLocation L, 1665 unsigned SubModuleID); 1666 1667 /// \brief Turn the specified lexer token into a fully checked and spelled 1668 /// filename, e.g. as an operand of \#include. 1669 /// 1670 /// The caller is expected to provide a buffer that is large enough to hold 1671 /// the spelling of the filename, but is also expected to handle the case 1672 /// when this method decides to use a different buffer. 1673 /// 1674 /// \returns true if the input filename was in <>'s or false if it was 1675 /// in ""'s. 1676 bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Filename); 1677 1678 /// \brief Given a "foo" or \<foo> reference, look up the indicated file. 1679 /// 1680 /// Returns null on failure. \p isAngled indicates whether the file 1681 /// reference is for system \#include's or not (i.e. using <> instead of ""). 1682 const FileEntry *LookupFile(SourceLocation FilenameLoc, StringRef Filename, 1683 bool isAngled, const DirectoryLookup *FromDir, 1684 const FileEntry *FromFile, 1685 const DirectoryLookup *&CurDir, 1686 SmallVectorImpl<char> *SearchPath, 1687 SmallVectorImpl<char> *RelativePath, 1688 ModuleMap::KnownHeader *SuggestedModule, 1689 bool SkipCache = false); 1690 1691 /// \brief Get the DirectoryLookup structure used to find the current 1692 /// FileEntry, if CurLexer is non-null and if applicable. 1693 /// 1694 /// This allows us to implement \#include_next and find directory-specific 1695 /// properties. 1696 const DirectoryLookup *GetCurDirLookup() { return CurDirLookup; } 1697 1698 /// \brief Return true if we're in the top-level file, not in a \#include. 1699 bool isInPrimaryFile() const; 1700 1701 /// \brief Handle cases where the \#include name is expanded 1702 /// from a macro as multiple tokens, which need to be glued together. 1703 /// 1704 /// This occurs for code like: 1705 /// \code 1706 /// \#define FOO <x/y.h> 1707 /// \#include FOO 1708 /// \endcode 1709 /// because in this case, "<x/y.h>" is returned as 7 tokens, not one. 1710 /// 1711 /// This code concatenates and consumes tokens up to the '>' token. It 1712 /// returns false if the > was found, otherwise it returns true if it finds 1713 /// and consumes the EOD marker. 1714 bool ConcatenateIncludeName(SmallString<128> &FilenameBuffer, 1715 SourceLocation &End); 1716 1717 /// \brief Lex an on-off-switch (C99 6.10.6p2) and verify that it is 1718 /// followed by EOD. Return true if the token is not a valid on-off-switch. 1719 bool LexOnOffSwitch(tok::OnOffSwitch &OOS); 1720 1721 bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef, 1722 bool *ShadowFlag = nullptr); 1723 1724 private: 1725 1726 void PushIncludeMacroStack() { 1727 assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer"); 1728 IncludeMacroStack.emplace_back( 1729 CurLexerKind, CurSubmodule, std::move(CurLexer), std::move(CurPTHLexer), 1730 CurPPLexer, std::move(CurTokenLexer), CurDirLookup); 1731 CurPPLexer = nullptr; 1732 } 1733 1734 void PopIncludeMacroStack() { 1735 CurLexer = std::move(IncludeMacroStack.back().TheLexer); 1736 CurPTHLexer = std::move(IncludeMacroStack.back().ThePTHLexer); 1737 CurPPLexer = IncludeMacroStack.back().ThePPLexer; 1738 CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer); 1739 CurDirLookup = IncludeMacroStack.back().TheDirLookup; 1740 CurSubmodule = IncludeMacroStack.back().TheSubmodule; 1741 CurLexerKind = IncludeMacroStack.back().CurLexerKind; 1742 IncludeMacroStack.pop_back(); 1743 } 1744 1745 void PropagateLineStartLeadingSpaceInfo(Token &Result); 1746 1747 void EnterSubmodule(Module *M, SourceLocation ImportLoc); 1748 void LeaveSubmodule(); 1749 1750 /// Determine whether we need to create module macros for #defines in the 1751 /// current context. 1752 bool needModuleMacros() const; 1753 1754 /// Update the set of active module macros and ambiguity flag for a module 1755 /// macro name. 1756 void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info); 1757 1758 /// \brief Allocate a new MacroInfo object. 1759 MacroInfo *AllocateMacroInfo(); 1760 1761 DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI, 1762 SourceLocation Loc); 1763 UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc); 1764 VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc, 1765 bool isPublic); 1766 1767 /// \brief Lex and validate a macro name, which occurs after a 1768 /// \#define or \#undef. 1769 /// 1770 /// \param MacroNameTok Token that represents the name defined or undefined. 1771 /// \param IsDefineUndef Kind if preprocessor directive. 1772 /// \param ShadowFlag Points to flag that is set if macro name shadows 1773 /// a keyword. 1774 /// 1775 /// This emits a diagnostic, sets the token kind to eod, 1776 /// and discards the rest of the macro line if the macro name is invalid. 1777 void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other, 1778 bool *ShadowFlag = nullptr); 1779 1780 /// The ( starting an argument list of a macro definition has just been read. 1781 /// Lex the rest of the arguments and the closing ), updating \p MI with 1782 /// what we learn and saving in \p LastTok the last token read. 1783 /// Return true if an error occurs parsing the arg list. 1784 bool ReadMacroDefinitionArgList(MacroInfo *MI, Token& LastTok); 1785 1786 /// We just read a \#if or related directive and decided that the 1787 /// subsequent tokens are in the \#if'd out portion of the 1788 /// file. Lex the rest of the file, until we see an \#endif. If \p 1789 /// FoundNonSkipPortion is true, then we have already emitted code for part of 1790 /// this \#if directive, so \#else/\#elif blocks should never be entered. If 1791 /// \p FoundElse is false, then \#else directives are ok, if not, then we have 1792 /// already seen one so a \#else directive is a duplicate. When this returns, 1793 /// the caller can lex the first valid token. 1794 void SkipExcludedConditionalBlock(SourceLocation IfTokenLoc, 1795 bool FoundNonSkipPortion, bool FoundElse, 1796 SourceLocation ElseLoc = SourceLocation()); 1797 1798 /// \brief A fast PTH version of SkipExcludedConditionalBlock. 1799 void PTHSkipExcludedConditionalBlock(); 1800 1801 /// \brief Evaluate an integer constant expression that may occur after a 1802 /// \#if or \#elif directive and return it as a bool. 1803 /// 1804 /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro. 1805 bool EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro); 1806 1807 /// \brief Install the standard preprocessor pragmas: 1808 /// \#pragma GCC poison/system_header/dependency and \#pragma once. 1809 void RegisterBuiltinPragmas(); 1810 1811 /// \brief Register builtin macros such as __LINE__ with the identifier table. 1812 void RegisterBuiltinMacros(); 1813 1814 /// If an identifier token is read that is to be expanded as a macro, handle 1815 /// it and return the next token as 'Tok'. If we lexed a token, return true; 1816 /// otherwise the caller should lex again. 1817 bool HandleMacroExpandedIdentifier(Token &Tok, const MacroDefinition &MD); 1818 1819 /// \brief Cache macro expanded tokens for TokenLexers. 1820 // 1821 /// Works like a stack; a TokenLexer adds the macro expanded tokens that is 1822 /// going to lex in the cache and when it finishes the tokens are removed 1823 /// from the end of the cache. 1824 Token *cacheMacroExpandedTokens(TokenLexer *tokLexer, 1825 ArrayRef<Token> tokens); 1826 void removeCachedMacroExpandedTokensOfLastLexer(); 1827 friend void TokenLexer::ExpandFunctionArguments(); 1828 1829 /// Determine whether the next preprocessor token to be 1830 /// lexed is a '('. If so, consume the token and return true, if not, this 1831 /// method should have no observable side-effect on the lexed tokens. 1832 bool isNextPPTokenLParen(); 1833 1834 /// After reading "MACRO(", this method is invoked to read all of the formal 1835 /// arguments specified for the macro invocation. Returns null on error. 1836 MacroArgs *ReadFunctionLikeMacroArgs(Token &MacroName, MacroInfo *MI, 1837 SourceLocation &ExpansionEnd); 1838 1839 /// \brief If an identifier token is read that is to be expanded 1840 /// as a builtin macro, handle it and return the next token as 'Tok'. 1841 void ExpandBuiltinMacro(Token &Tok); 1842 1843 /// \brief Read a \c _Pragma directive, slice it up, process it, then 1844 /// return the first token after the directive. 1845 /// This assumes that the \c _Pragma token has just been read into \p Tok. 1846 void Handle_Pragma(Token &Tok); 1847 1848 /// \brief Like Handle_Pragma except the pragma text is not enclosed within 1849 /// a string literal. 1850 void HandleMicrosoft__pragma(Token &Tok); 1851 1852 /// \brief Add a lexer to the top of the include stack and 1853 /// start lexing tokens from it instead of the current buffer. 1854 void EnterSourceFileWithLexer(Lexer *TheLexer, const DirectoryLookup *Dir); 1855 1856 /// \brief Add a lexer to the top of the include stack and 1857 /// start getting tokens from it using the PTH cache. 1858 void EnterSourceFileWithPTH(PTHLexer *PL, const DirectoryLookup *Dir); 1859 1860 /// \brief Set the FileID for the preprocessor predefines. 1861 void setPredefinesFileID(FileID FID) { 1862 assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!"); 1863 PredefinesFileID = FID; 1864 } 1865 1866 /// \brief Returns true if we are lexing from a file and not a 1867 /// pragma or a macro. 1868 static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) { 1869 return L ? !L->isPragmaLexer() : P != nullptr; 1870 } 1871 1872 static bool IsFileLexer(const IncludeStackInfo& I) { 1873 return IsFileLexer(I.TheLexer.get(), I.ThePPLexer); 1874 } 1875 1876 bool IsFileLexer() const { 1877 return IsFileLexer(CurLexer.get(), CurPPLexer); 1878 } 1879 1880 //===--------------------------------------------------------------------===// 1881 // Caching stuff. 1882 void CachingLex(Token &Result); 1883 bool InCachingLexMode() const { 1884 // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means 1885 // that we are past EOF, not that we are in CachingLex mode. 1886 return !CurPPLexer && !CurTokenLexer && !CurPTHLexer && 1887 !IncludeMacroStack.empty(); 1888 } 1889 void EnterCachingLexMode(); 1890 void ExitCachingLexMode() { 1891 if (InCachingLexMode()) 1892 RemoveTopOfLexerStack(); 1893 } 1894 const Token &PeekAhead(unsigned N); 1895 void AnnotatePreviousCachedTokens(const Token &Tok); 1896 1897 //===--------------------------------------------------------------------===// 1898 /// Handle*Directive - implement the various preprocessor directives. These 1899 /// should side-effect the current preprocessor object so that the next call 1900 /// to Lex() will return the appropriate token next. 1901 void HandleLineDirective(); 1902 void HandleDigitDirective(Token &Tok); 1903 void HandleUserDiagnosticDirective(Token &Tok, bool isWarning); 1904 void HandleIdentSCCSDirective(Token &Tok); 1905 void HandleMacroPublicDirective(Token &Tok); 1906 void HandleMacroPrivateDirective(); 1907 1908 // File inclusion. 1909 void HandleIncludeDirective(SourceLocation HashLoc, 1910 Token &Tok, 1911 const DirectoryLookup *LookupFrom = nullptr, 1912 const FileEntry *LookupFromFile = nullptr, 1913 bool isImport = false); 1914 void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok); 1915 void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok); 1916 void HandleImportDirective(SourceLocation HashLoc, Token &Tok); 1917 void HandleMicrosoftImportDirective(Token &Tok); 1918 1919 public: 1920 // Module inclusion testing. 1921 /// \brief Find the module that owns the source or header file that 1922 /// \p Loc points to. If the location is in a file that was included 1923 /// into a module, or is outside any module, returns nullptr. 1924 Module *getModuleForLocation(SourceLocation Loc); 1925 1926 /// \brief Find the module that contains the specified location, either 1927 /// directly or indirectly. 1928 Module *getModuleContainingLocation(SourceLocation Loc); 1929 1930 /// \brief We want to produce a diagnostic at location IncLoc concerning a 1931 /// missing module import. 1932 /// 1933 /// \param IncLoc The location at which the missing import was detected. 1934 /// \param MLoc A location within the desired module at which some desired 1935 /// effect occurred (eg, where a desired entity was declared). 1936 /// 1937 /// \return A file that can be #included to import a module containing MLoc. 1938 /// Null if no such file could be determined or if a #include is not 1939 /// appropriate. 1940 const FileEntry *getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc, 1941 SourceLocation MLoc); 1942 1943 private: 1944 // Macro handling. 1945 void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterTopLevelIfndef); 1946 void HandleUndefDirective(); 1947 1948 // Conditional Inclusion. 1949 void HandleIfdefDirective(Token &Tok, bool isIfndef, 1950 bool ReadAnyTokensBeforeDirective); 1951 void HandleIfDirective(Token &Tok, bool ReadAnyTokensBeforeDirective); 1952 void HandleEndifDirective(Token &Tok); 1953 void HandleElseDirective(Token &Tok); 1954 void HandleElifDirective(Token &Tok); 1955 1956 // Pragmas. 1957 void HandlePragmaDirective(SourceLocation IntroducerLoc, 1958 PragmaIntroducerKind Introducer); 1959 public: 1960 void HandlePragmaOnce(Token &OnceTok); 1961 void HandlePragmaMark(); 1962 void HandlePragmaPoison(); 1963 void HandlePragmaSystemHeader(Token &SysHeaderTok); 1964 void HandlePragmaDependency(Token &DependencyTok); 1965 void HandlePragmaPushMacro(Token &Tok); 1966 void HandlePragmaPopMacro(Token &Tok); 1967 void HandlePragmaIncludeAlias(Token &Tok); 1968 IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok); 1969 1970 // Return true and store the first token only if any CommentHandler 1971 // has inserted some tokens and getCommentRetentionState() is false. 1972 bool HandleComment(Token &Token, SourceRange Comment); 1973 1974 /// \brief A macro is used, update information about macros that need unused 1975 /// warnings. 1976 void markMacroAsUsed(MacroInfo *MI); 1977 }; 1978 1979 /// \brief Abstract base class that describes a handler that will receive 1980 /// source ranges for each of the comments encountered in the source file. 1981 class CommentHandler { 1982 public: 1983 virtual ~CommentHandler(); 1984 1985 // The handler shall return true if it has pushed any tokens 1986 // to be read using e.g. EnterToken or EnterTokenStream. 1987 virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0; 1988 }; 1989 1990 /// \brief Registry of pragma handlers added by plugins 1991 typedef llvm::Registry<PragmaHandler> PragmaHandlerRegistry; 1992 1993 } // end namespace clang 1994 1995 #endif 1996