Home | History | Annotate | Download | only in Lex
      1 //===--- PTHLexer.cpp - Lex from a token stream ---------------------------===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 // This file implements the PTHLexer interface.
     11 //
     12 //===----------------------------------------------------------------------===//
     13 
     14 #include "clang/Lex/PTHLexer.h"
     15 #include "clang/Basic/FileManager.h"
     16 #include "clang/Basic/FileSystemStatCache.h"
     17 #include "clang/Basic/IdentifierTable.h"
     18 #include "clang/Basic/OnDiskHashTable.h"
     19 #include "clang/Basic/TokenKinds.h"
     20 #include "clang/Lex/LexDiagnostic.h"
     21 #include "clang/Lex/PTHManager.h"
     22 #include "clang/Lex/Preprocessor.h"
     23 #include "clang/Lex/Token.h"
     24 #include "llvm/ADT/OwningPtr.h"
     25 #include "llvm/ADT/StringExtras.h"
     26 #include "llvm/ADT/StringMap.h"
     27 #include "llvm/Support/MemoryBuffer.h"
     28 #include "llvm/Support/system_error.h"
     29 using namespace clang;
     30 using namespace clang::io;
     31 
     32 #define DISK_TOKEN_SIZE (1+1+2+4+4)
     33 
     34 //===----------------------------------------------------------------------===//
     35 // PTHLexer methods.
     36 //===----------------------------------------------------------------------===//
     37 
     38 PTHLexer::PTHLexer(Preprocessor &PP, FileID FID, const unsigned char *D,
     39                    const unsigned char *ppcond, PTHManager &PM)
     40   : PreprocessorLexer(&PP, FID), TokBuf(D), CurPtr(D), LastHashTokPtr(0),
     41     PPCond(ppcond), CurPPCondPtr(ppcond), PTHMgr(PM) {
     42 
     43   FileStartLoc = PP.getSourceManager().getLocForStartOfFile(FID);
     44 }
     45 
     46 void PTHLexer::Lex(Token& Tok) {
     47 LexNextToken:
     48 
     49   //===--------------------------------------==//
     50   // Read the raw token data.
     51   //===--------------------------------------==//
     52 
     53   // Shadow CurPtr into an automatic variable.
     54   const unsigned char *CurPtrShadow = CurPtr;
     55 
     56   // Read in the data for the token.
     57   unsigned Word0 = ReadLE32(CurPtrShadow);
     58   uint32_t IdentifierID = ReadLE32(CurPtrShadow);
     59   uint32_t FileOffset = ReadLE32(CurPtrShadow);
     60 
     61   tok::TokenKind TKind = (tok::TokenKind) (Word0 & 0xFF);
     62   Token::TokenFlags TFlags = (Token::TokenFlags) ((Word0 >> 8) & 0xFF);
     63   uint32_t Len = Word0 >> 16;
     64 
     65   CurPtr = CurPtrShadow;
     66 
     67   //===--------------------------------------==//
     68   // Construct the token itself.
     69   //===--------------------------------------==//
     70 
     71   Tok.startToken();
     72   Tok.setKind(TKind);
     73   Tok.setFlag(TFlags);
     74   assert(!LexingRawMode);
     75   Tok.setLocation(FileStartLoc.getLocWithOffset(FileOffset));
     76   Tok.setLength(Len);
     77 
     78   // Handle identifiers.
     79   if (Tok.isLiteral()) {
     80     Tok.setLiteralData((const char*) (PTHMgr.SpellingBase + IdentifierID));
     81   }
     82   else if (IdentifierID) {
     83     MIOpt.ReadToken();
     84     IdentifierInfo *II = PTHMgr.GetIdentifierInfo(IdentifierID-1);
     85 
     86     Tok.setIdentifierInfo(II);
     87 
     88     // Change the kind of this identifier to the appropriate token kind, e.g.
     89     // turning "for" into a keyword.
     90     Tok.setKind(II->getTokenID());
     91 
     92     if (II->isHandleIdentifierCase())
     93       PP->HandleIdentifier(Tok);
     94     return;
     95   }
     96 
     97   //===--------------------------------------==//
     98   // Process the token.
     99   //===--------------------------------------==//
    100   if (TKind == tok::eof) {
    101     // Save the end-of-file token.
    102     EofToken = Tok;
    103 
    104     // Save 'PP' to 'PPCache' as LexEndOfFile can delete 'this'.
    105     Preprocessor *PPCache = PP;
    106 
    107     assert(!ParsingPreprocessorDirective);
    108     assert(!LexingRawMode);
    109 
    110     if (LexEndOfFile(Tok))
    111       return;
    112 
    113     return PPCache->Lex(Tok);
    114   }
    115 
    116   if (TKind == tok::hash && Tok.isAtStartOfLine()) {
    117     LastHashTokPtr = CurPtr - DISK_TOKEN_SIZE;
    118     assert(!LexingRawMode);
    119     PP->HandleDirective(Tok);
    120 
    121     if (PP->isCurrentLexer(this))
    122       goto LexNextToken;
    123 
    124     return PP->Lex(Tok);
    125   }
    126 
    127   if (TKind == tok::eod) {
    128     assert(ParsingPreprocessorDirective);
    129     ParsingPreprocessorDirective = false;
    130     return;
    131   }
    132 
    133   MIOpt.ReadToken();
    134 }
    135 
    136 bool PTHLexer::LexEndOfFile(Token &Result) {
    137   // If we hit the end of the file while parsing a preprocessor directive,
    138   // end the preprocessor directive first.  The next token returned will
    139   // then be the end of file.
    140   if (ParsingPreprocessorDirective) {
    141     ParsingPreprocessorDirective = false; // Done parsing the "line".
    142     return true;  // Have a token.
    143   }
    144 
    145   assert(!LexingRawMode);
    146 
    147   // If we are in a #if directive, emit an error.
    148   while (!ConditionalStack.empty()) {
    149     if (PP->getCodeCompletionFileLoc() != FileStartLoc)
    150       PP->Diag(ConditionalStack.back().IfLoc,
    151                diag::err_pp_unterminated_conditional);
    152     ConditionalStack.pop_back();
    153   }
    154 
    155   // Finally, let the preprocessor handle this.
    156   return PP->HandleEndOfFile(Result);
    157 }
    158 
    159 // FIXME: We can just grab the last token instead of storing a copy
    160 // into EofToken.
    161 void PTHLexer::getEOF(Token& Tok) {
    162   assert(EofToken.is(tok::eof));
    163   Tok = EofToken;
    164 }
    165 
    166 void PTHLexer::DiscardToEndOfLine() {
    167   assert(ParsingPreprocessorDirective && ParsingFilename == false &&
    168          "Must be in a preprocessing directive!");
    169 
    170   // We assume that if the preprocessor wishes to discard to the end of
    171   // the line that it also means to end the current preprocessor directive.
    172   ParsingPreprocessorDirective = false;
    173 
    174   // Skip tokens by only peeking at their token kind and the flags.
    175   // We don't need to actually reconstruct full tokens from the token buffer.
    176   // This saves some copies and it also reduces IdentifierInfo* lookup.
    177   const unsigned char* p = CurPtr;
    178   while (1) {
    179     // Read the token kind.  Are we at the end of the file?
    180     tok::TokenKind x = (tok::TokenKind) (uint8_t) *p;
    181     if (x == tok::eof) break;
    182 
    183     // Read the token flags.  Are we at the start of the next line?
    184     Token::TokenFlags y = (Token::TokenFlags) (uint8_t) p[1];
    185     if (y & Token::StartOfLine) break;
    186 
    187     // Skip to the next token.
    188     p += DISK_TOKEN_SIZE;
    189   }
    190 
    191   CurPtr = p;
    192 }
    193 
    194 /// SkipBlock - Used by Preprocessor to skip the current conditional block.
    195 bool PTHLexer::SkipBlock() {
    196   assert(CurPPCondPtr && "No cached PP conditional information.");
    197   assert(LastHashTokPtr && "No known '#' token.");
    198 
    199   const unsigned char* HashEntryI = 0;
    200   uint32_t TableIdx;
    201 
    202   do {
    203     // Read the token offset from the side-table.
    204     uint32_t Offset = ReadLE32(CurPPCondPtr);
    205 
    206     // Read the target table index from the side-table.
    207     TableIdx = ReadLE32(CurPPCondPtr);
    208 
    209     // Compute the actual memory address of the '#' token data for this entry.
    210     HashEntryI = TokBuf + Offset;
    211 
    212     // Optmization: "Sibling jumping".  #if...#else...#endif blocks can
    213     //  contain nested blocks.  In the side-table we can jump over these
    214     //  nested blocks instead of doing a linear search if the next "sibling"
    215     //  entry is not at a location greater than LastHashTokPtr.
    216     if (HashEntryI < LastHashTokPtr && TableIdx) {
    217       // In the side-table we are still at an entry for a '#' token that
    218       // is earlier than the last one we saw.  Check if the location we would
    219       // stride gets us closer.
    220       const unsigned char* NextPPCondPtr =
    221         PPCond + TableIdx*(sizeof(uint32_t)*2);
    222       assert(NextPPCondPtr >= CurPPCondPtr);
    223       // Read where we should jump to.
    224       const unsigned char* HashEntryJ = TokBuf + ReadLE32(NextPPCondPtr);
    225 
    226       if (HashEntryJ <= LastHashTokPtr) {
    227         // Jump directly to the next entry in the side table.
    228         HashEntryI = HashEntryJ;
    229         TableIdx = ReadLE32(NextPPCondPtr);
    230         CurPPCondPtr = NextPPCondPtr;
    231       }
    232     }
    233   }
    234   while (HashEntryI < LastHashTokPtr);
    235   assert(HashEntryI == LastHashTokPtr && "No PP-cond entry found for '#'");
    236   assert(TableIdx && "No jumping from #endifs.");
    237 
    238   // Update our side-table iterator.
    239   const unsigned char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2);
    240   assert(NextPPCondPtr >= CurPPCondPtr);
    241   CurPPCondPtr = NextPPCondPtr;
    242 
    243   // Read where we should jump to.
    244   HashEntryI = TokBuf + ReadLE32(NextPPCondPtr);
    245   uint32_t NextIdx = ReadLE32(NextPPCondPtr);
    246 
    247   // By construction NextIdx will be zero if this is a #endif.  This is useful
    248   // to know to obviate lexing another token.
    249   bool isEndif = NextIdx == 0;
    250 
    251   // This case can occur when we see something like this:
    252   //
    253   //  #if ...
    254   //   /* a comment or nothing */
    255   //  #elif
    256   //
    257   // If we are skipping the first #if block it will be the case that CurPtr
    258   // already points 'elif'.  Just return.
    259 
    260   if (CurPtr > HashEntryI) {
    261     assert(CurPtr == HashEntryI + DISK_TOKEN_SIZE);
    262     // Did we reach a #endif?  If so, go ahead and consume that token as well.
    263     if (isEndif)
    264       CurPtr += DISK_TOKEN_SIZE*2;
    265     else
    266       LastHashTokPtr = HashEntryI;
    267 
    268     return isEndif;
    269   }
    270 
    271   // Otherwise, we need to advance.  Update CurPtr to point to the '#' token.
    272   CurPtr = HashEntryI;
    273 
    274   // Update the location of the last observed '#'.  This is useful if we
    275   // are skipping multiple blocks.
    276   LastHashTokPtr = CurPtr;
    277 
    278   // Skip the '#' token.
    279   assert(((tok::TokenKind)*CurPtr) == tok::hash);
    280   CurPtr += DISK_TOKEN_SIZE;
    281 
    282   // Did we reach a #endif?  If so, go ahead and consume that token as well.
    283   if (isEndif) { CurPtr += DISK_TOKEN_SIZE*2; }
    284 
    285   return isEndif;
    286 }
    287 
    288 SourceLocation PTHLexer::getSourceLocation() {
    289   // getSourceLocation is not on the hot path.  It is used to get the location
    290   // of the next token when transitioning back to this lexer when done
    291   // handling a #included file.  Just read the necessary data from the token
    292   // data buffer to construct the SourceLocation object.
    293   // NOTE: This is a virtual function; hence it is defined out-of-line.
    294   const unsigned char *OffsetPtr = CurPtr + (DISK_TOKEN_SIZE - 4);
    295   uint32_t Offset = ReadLE32(OffsetPtr);
    296   return FileStartLoc.getLocWithOffset(Offset);
    297 }
    298 
    299 //===----------------------------------------------------------------------===//
    300 // PTH file lookup: map from strings to file data.
    301 //===----------------------------------------------------------------------===//
    302 
    303 /// PTHFileLookup - This internal data structure is used by the PTHManager
    304 ///  to map from FileEntry objects managed by FileManager to offsets within
    305 ///  the PTH file.
    306 namespace {
    307 class PTHFileData {
    308   const uint32_t TokenOff;
    309   const uint32_t PPCondOff;
    310 public:
    311   PTHFileData(uint32_t tokenOff, uint32_t ppCondOff)
    312     : TokenOff(tokenOff), PPCondOff(ppCondOff) {}
    313 
    314   uint32_t getTokenOffset() const { return TokenOff; }
    315   uint32_t getPPCondOffset() const { return PPCondOff; }
    316 };
    317 
    318 
    319 class PTHFileLookupCommonTrait {
    320 public:
    321   typedef std::pair<unsigned char, const char*> internal_key_type;
    322 
    323   static unsigned ComputeHash(internal_key_type x) {
    324     return llvm::HashString(x.second);
    325   }
    326 
    327   static std::pair<unsigned, unsigned>
    328   ReadKeyDataLength(const unsigned char*& d) {
    329     unsigned keyLen = (unsigned) ReadUnalignedLE16(d);
    330     unsigned dataLen = (unsigned) *(d++);
    331     return std::make_pair(keyLen, dataLen);
    332   }
    333 
    334   static internal_key_type ReadKey(const unsigned char* d, unsigned) {
    335     unsigned char k = *(d++); // Read the entry kind.
    336     return std::make_pair(k, (const char*) d);
    337   }
    338 };
    339 
    340 class PTHFileLookupTrait : public PTHFileLookupCommonTrait {
    341 public:
    342   typedef const FileEntry* external_key_type;
    343   typedef PTHFileData      data_type;
    344 
    345   static internal_key_type GetInternalKey(const FileEntry* FE) {
    346     return std::make_pair((unsigned char) 0x1, FE->getName());
    347   }
    348 
    349   static bool EqualKey(internal_key_type a, internal_key_type b) {
    350     return a.first == b.first && strcmp(a.second, b.second) == 0;
    351   }
    352 
    353   static PTHFileData ReadData(const internal_key_type& k,
    354                               const unsigned char* d, unsigned) {
    355     assert(k.first == 0x1 && "Only file lookups can match!");
    356     uint32_t x = ::ReadUnalignedLE32(d);
    357     uint32_t y = ::ReadUnalignedLE32(d);
    358     return PTHFileData(x, y);
    359   }
    360 };
    361 
    362 class PTHStringLookupTrait {
    363 public:
    364   typedef uint32_t
    365           data_type;
    366 
    367   typedef const std::pair<const char*, unsigned>
    368           external_key_type;
    369 
    370   typedef external_key_type internal_key_type;
    371 
    372   static bool EqualKey(const internal_key_type& a,
    373                        const internal_key_type& b) {
    374     return (a.second == b.second) ? memcmp(a.first, b.first, a.second) == 0
    375                                   : false;
    376   }
    377 
    378   static unsigned ComputeHash(const internal_key_type& a) {
    379     return llvm::HashString(StringRef(a.first, a.second));
    380   }
    381 
    382   // This hopefully will just get inlined and removed by the optimizer.
    383   static const internal_key_type&
    384   GetInternalKey(const external_key_type& x) { return x; }
    385 
    386   static std::pair<unsigned, unsigned>
    387   ReadKeyDataLength(const unsigned char*& d) {
    388     return std::make_pair((unsigned) ReadUnalignedLE16(d), sizeof(uint32_t));
    389   }
    390 
    391   static std::pair<const char*, unsigned>
    392   ReadKey(const unsigned char* d, unsigned n) {
    393       assert(n >= 2 && d[n-1] == '\0');
    394       return std::make_pair((const char*) d, n-1);
    395     }
    396 
    397   static uint32_t ReadData(const internal_key_type& k, const unsigned char* d,
    398                            unsigned) {
    399     return ::ReadUnalignedLE32(d);
    400   }
    401 };
    402 
    403 } // end anonymous namespace
    404 
    405 typedef OnDiskChainedHashTable<PTHFileLookupTrait>   PTHFileLookup;
    406 typedef OnDiskChainedHashTable<PTHStringLookupTrait> PTHStringIdLookup;
    407 
    408 //===----------------------------------------------------------------------===//
    409 // PTHManager methods.
    410 //===----------------------------------------------------------------------===//
    411 
    412 PTHManager::PTHManager(const llvm::MemoryBuffer* buf, void* fileLookup,
    413                        const unsigned char* idDataTable,
    414                        IdentifierInfo** perIDCache,
    415                        void* stringIdLookup, unsigned numIds,
    416                        const unsigned char* spellingBase,
    417                        const char* originalSourceFile)
    418 : Buf(buf), PerIDCache(perIDCache), FileLookup(fileLookup),
    419   IdDataTable(idDataTable), StringIdLookup(stringIdLookup),
    420   NumIds(numIds), PP(0), SpellingBase(spellingBase),
    421   OriginalSourceFile(originalSourceFile) {}
    422 
    423 PTHManager::~PTHManager() {
    424   delete Buf;
    425   delete (PTHFileLookup*) FileLookup;
    426   delete (PTHStringIdLookup*) StringIdLookup;
    427   free(PerIDCache);
    428 }
    429 
    430 static void InvalidPTH(DiagnosticsEngine &Diags, const char *Msg) {
    431   Diags.Report(Diags.getCustomDiagID(DiagnosticsEngine::Error, Msg));
    432 }
    433 
    434 PTHManager *PTHManager::Create(const std::string &file,
    435                                DiagnosticsEngine &Diags) {
    436   // Memory map the PTH file.
    437   OwningPtr<llvm::MemoryBuffer> File;
    438 
    439   if (llvm::MemoryBuffer::getFile(file, File)) {
    440     // FIXME: Add ec.message() to this diag.
    441     Diags.Report(diag::err_invalid_pth_file) << file;
    442     return 0;
    443   }
    444 
    445   // Get the buffer ranges and check if there are at least three 32-bit
    446   // words at the end of the file.
    447   const unsigned char *BufBeg = (const unsigned char*)File->getBufferStart();
    448   const unsigned char *BufEnd = (const unsigned char*)File->getBufferEnd();
    449 
    450   // Check the prologue of the file.
    451   if ((BufEnd - BufBeg) < (signed)(sizeof("cfe-pth") + 4 + 4) ||
    452       memcmp(BufBeg, "cfe-pth", sizeof("cfe-pth")) != 0) {
    453     Diags.Report(diag::err_invalid_pth_file) << file;
    454     return 0;
    455   }
    456 
    457   // Read the PTH version.
    458   const unsigned char *p = BufBeg + (sizeof("cfe-pth"));
    459   unsigned Version = ReadLE32(p);
    460 
    461   if (Version < PTHManager::Version) {
    462     InvalidPTH(Diags,
    463         Version < PTHManager::Version
    464         ? "PTH file uses an older PTH format that is no longer supported"
    465         : "PTH file uses a newer PTH format that cannot be read");
    466     return 0;
    467   }
    468 
    469   // Compute the address of the index table at the end of the PTH file.
    470   const unsigned char *PrologueOffset = p;
    471 
    472   if (PrologueOffset >= BufEnd) {
    473     Diags.Report(diag::err_invalid_pth_file) << file;
    474     return 0;
    475   }
    476 
    477   // Construct the file lookup table.  This will be used for mapping from
    478   // FileEntry*'s to cached tokens.
    479   const unsigned char* FileTableOffset = PrologueOffset + sizeof(uint32_t)*2;
    480   const unsigned char* FileTable = BufBeg + ReadLE32(FileTableOffset);
    481 
    482   if (!(FileTable > BufBeg && FileTable < BufEnd)) {
    483     Diags.Report(diag::err_invalid_pth_file) << file;
    484     return 0; // FIXME: Proper error diagnostic?
    485   }
    486 
    487   OwningPtr<PTHFileLookup> FL(PTHFileLookup::Create(FileTable, BufBeg));
    488 
    489   // Warn if the PTH file is empty.  We still want to create a PTHManager
    490   // as the PTH could be used with -include-pth.
    491   if (FL->isEmpty())
    492     InvalidPTH(Diags, "PTH file contains no cached source data");
    493 
    494   // Get the location of the table mapping from persistent ids to the
    495   // data needed to reconstruct identifiers.
    496   const unsigned char* IDTableOffset = PrologueOffset + sizeof(uint32_t)*0;
    497   const unsigned char* IData = BufBeg + ReadLE32(IDTableOffset);
    498 
    499   if (!(IData >= BufBeg && IData < BufEnd)) {
    500     Diags.Report(diag::err_invalid_pth_file) << file;
    501     return 0;
    502   }
    503 
    504   // Get the location of the hashtable mapping between strings and
    505   // persistent IDs.
    506   const unsigned char* StringIdTableOffset = PrologueOffset + sizeof(uint32_t)*1;
    507   const unsigned char* StringIdTable = BufBeg + ReadLE32(StringIdTableOffset);
    508   if (!(StringIdTable >= BufBeg && StringIdTable < BufEnd)) {
    509     Diags.Report(diag::err_invalid_pth_file) << file;
    510     return 0;
    511   }
    512 
    513   OwningPtr<PTHStringIdLookup> SL(PTHStringIdLookup::Create(StringIdTable,
    514                                                                   BufBeg));
    515 
    516   // Get the location of the spelling cache.
    517   const unsigned char* spellingBaseOffset = PrologueOffset + sizeof(uint32_t)*3;
    518   const unsigned char* spellingBase = BufBeg + ReadLE32(spellingBaseOffset);
    519   if (!(spellingBase >= BufBeg && spellingBase < BufEnd)) {
    520     Diags.Report(diag::err_invalid_pth_file) << file;
    521     return 0;
    522   }
    523 
    524   // Get the number of IdentifierInfos and pre-allocate the identifier cache.
    525   uint32_t NumIds = ReadLE32(IData);
    526 
    527   // Pre-allocate the persistent ID -> IdentifierInfo* cache.  We use calloc()
    528   // so that we in the best case only zero out memory once when the OS returns
    529   // us new pages.
    530   IdentifierInfo** PerIDCache = 0;
    531 
    532   if (NumIds) {
    533     PerIDCache = (IdentifierInfo**)calloc(NumIds, sizeof(*PerIDCache));
    534     if (!PerIDCache) {
    535       InvalidPTH(Diags, "Could not allocate memory for processing PTH file");
    536       return 0;
    537     }
    538   }
    539 
    540   // Compute the address of the original source file.
    541   const unsigned char* originalSourceBase = PrologueOffset + sizeof(uint32_t)*4;
    542   unsigned len = ReadUnalignedLE16(originalSourceBase);
    543   if (!len) originalSourceBase = 0;
    544 
    545   // Create the new PTHManager.
    546   return new PTHManager(File.take(), FL.take(), IData, PerIDCache,
    547                         SL.take(), NumIds, spellingBase,
    548                         (const char*) originalSourceBase);
    549 }
    550 
    551 IdentifierInfo* PTHManager::LazilyCreateIdentifierInfo(unsigned PersistentID) {
    552   // Look in the PTH file for the string data for the IdentifierInfo object.
    553   const unsigned char* TableEntry = IdDataTable + sizeof(uint32_t)*PersistentID;
    554   const unsigned char* IDData =
    555     (const unsigned char*)Buf->getBufferStart() + ReadLE32(TableEntry);
    556   assert(IDData < (const unsigned char*)Buf->getBufferEnd());
    557 
    558   // Allocate the object.
    559   std::pair<IdentifierInfo,const unsigned char*> *Mem =
    560     Alloc.Allocate<std::pair<IdentifierInfo,const unsigned char*> >();
    561 
    562   Mem->second = IDData;
    563   assert(IDData[0] != '\0');
    564   IdentifierInfo *II = new ((void*) Mem) IdentifierInfo();
    565 
    566   // Store the new IdentifierInfo in the cache.
    567   PerIDCache[PersistentID] = II;
    568   assert(II->getNameStart() && II->getNameStart()[0] != '\0');
    569   return II;
    570 }
    571 
    572 IdentifierInfo* PTHManager::get(StringRef Name) {
    573   PTHStringIdLookup& SL = *((PTHStringIdLookup*)StringIdLookup);
    574   // Double check our assumption that the last character isn't '\0'.
    575   assert(Name.empty() || Name.back() != '\0');
    576   PTHStringIdLookup::iterator I = SL.find(std::make_pair(Name.data(),
    577                                                          Name.size()));
    578   if (I == SL.end()) // No identifier found?
    579     return 0;
    580 
    581   // Match found.  Return the identifier!
    582   assert(*I > 0);
    583   return GetIdentifierInfo(*I-1);
    584 }
    585 
    586 PTHLexer *PTHManager::CreateLexer(FileID FID) {
    587   const FileEntry *FE = PP->getSourceManager().getFileEntryForID(FID);
    588   if (!FE)
    589     return 0;
    590 
    591   // Lookup the FileEntry object in our file lookup data structure.  It will
    592   // return a variant that indicates whether or not there is an offset within
    593   // the PTH file that contains cached tokens.
    594   PTHFileLookup& PFL = *((PTHFileLookup*)FileLookup);
    595   PTHFileLookup::iterator I = PFL.find(FE);
    596 
    597   if (I == PFL.end()) // No tokens available?
    598     return 0;
    599 
    600   const PTHFileData& FileData = *I;
    601 
    602   const unsigned char *BufStart = (const unsigned char *)Buf->getBufferStart();
    603   // Compute the offset of the token data within the buffer.
    604   const unsigned char* data = BufStart + FileData.getTokenOffset();
    605 
    606   // Get the location of pp-conditional table.
    607   const unsigned char* ppcond = BufStart + FileData.getPPCondOffset();
    608   uint32_t Len = ReadLE32(ppcond);
    609   if (Len == 0) ppcond = 0;
    610 
    611   assert(PP && "No preprocessor set yet!");
    612   return new PTHLexer(*PP, FID, data, ppcond, *this);
    613 }
    614 
    615 //===----------------------------------------------------------------------===//
    616 // 'stat' caching.
    617 //===----------------------------------------------------------------------===//
    618 
    619 namespace {
    620 class PTHStatData {
    621 public:
    622   const bool hasStat;
    623   const ino_t ino;
    624   const dev_t dev;
    625   const mode_t mode;
    626   const time_t mtime;
    627   const off_t size;
    628 
    629   PTHStatData(ino_t i, dev_t d, mode_t mo, time_t m, off_t s)
    630   : hasStat(true), ino(i), dev(d), mode(mo), mtime(m), size(s) {}
    631 
    632   PTHStatData()
    633     : hasStat(false), ino(0), dev(0), mode(0), mtime(0), size(0) {}
    634 };
    635 
    636 class PTHStatLookupTrait : public PTHFileLookupCommonTrait {
    637 public:
    638   typedef const char* external_key_type;  // const char*
    639   typedef PTHStatData data_type;
    640 
    641   static internal_key_type GetInternalKey(const char *path) {
    642     // The key 'kind' doesn't matter here because it is ignored in EqualKey.
    643     return std::make_pair((unsigned char) 0x0, path);
    644   }
    645 
    646   static bool EqualKey(internal_key_type a, internal_key_type b) {
    647     // When doing 'stat' lookups we don't care about the kind of 'a' and 'b',
    648     // just the paths.
    649     return strcmp(a.second, b.second) == 0;
    650   }
    651 
    652   static data_type ReadData(const internal_key_type& k, const unsigned char* d,
    653                             unsigned) {
    654 
    655     if (k.first /* File or Directory */) {
    656       if (k.first == 0x1 /* File */) d += 4 * 2; // Skip the first 2 words.
    657       ino_t ino = (ino_t) ReadUnalignedLE32(d);
    658       dev_t dev = (dev_t) ReadUnalignedLE32(d);
    659       mode_t mode = (mode_t) ReadUnalignedLE16(d);
    660       time_t mtime = (time_t) ReadUnalignedLE64(d);
    661       return data_type(ino, dev, mode, mtime, (off_t) ReadUnalignedLE64(d));
    662     }
    663 
    664     // Negative stat.  Don't read anything.
    665     return data_type();
    666   }
    667 };
    668 
    669 class PTHStatCache : public FileSystemStatCache {
    670   typedef OnDiskChainedHashTable<PTHStatLookupTrait> CacheTy;
    671   CacheTy Cache;
    672 
    673 public:
    674   PTHStatCache(PTHFileLookup &FL) :
    675     Cache(FL.getNumBuckets(), FL.getNumEntries(), FL.getBuckets(),
    676           FL.getBase()) {}
    677 
    678   ~PTHStatCache() {}
    679 
    680   LookupResult getStat(const char *Path, struct stat &StatBuf,
    681                        bool isFile, int *FileDescriptor) {
    682     // Do the lookup for the file's data in the PTH file.
    683     CacheTy::iterator I = Cache.find(Path);
    684 
    685     // If we don't get a hit in the PTH file just forward to 'stat'.
    686     if (I == Cache.end())
    687       return statChained(Path, StatBuf, isFile, FileDescriptor);
    688 
    689     const PTHStatData &Data = *I;
    690 
    691     if (!Data.hasStat)
    692       return CacheMissing;
    693 
    694     StatBuf.st_ino = Data.ino;
    695     StatBuf.st_dev = Data.dev;
    696     StatBuf.st_mtime = Data.mtime;
    697     StatBuf.st_mode = Data.mode;
    698     StatBuf.st_size = Data.size;
    699     return CacheExists;
    700   }
    701 };
    702 } // end anonymous namespace
    703 
    704 FileSystemStatCache *PTHManager::createStatCache() {
    705   return new PTHStatCache(*((PTHFileLookup*) FileLookup));
    706 }
    707