Home | History | Annotate | Download | only in Lex
      1 //===--- PTHLexer.cpp - Lex from a token stream ---------------------------===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 // This file implements the PTHLexer interface.
     11 //
     12 //===----------------------------------------------------------------------===//
     13 
     14 #include "clang/Basic/TokenKinds.h"
     15 #include "clang/Basic/FileManager.h"
     16 #include "clang/Basic/FileSystemStatCache.h"
     17 #include "clang/Basic/IdentifierTable.h"
     18 #include "clang/Basic/OnDiskHashTable.h"
     19 #include "clang/Lex/LexDiagnostic.h"
     20 #include "clang/Lex/PTHLexer.h"
     21 #include "clang/Lex/Preprocessor.h"
     22 #include "clang/Lex/PTHManager.h"
     23 #include "clang/Lex/Token.h"
     24 #include "clang/Lex/Preprocessor.h"
     25 #include "llvm/ADT/OwningPtr.h"
     26 #include "llvm/ADT/StringExtras.h"
     27 #include "llvm/ADT/StringMap.h"
     28 #include "llvm/Support/MemoryBuffer.h"
     29 #include "llvm/Support/system_error.h"
     30 using namespace clang;
     31 using namespace clang::io;
     32 
     33 #define DISK_TOKEN_SIZE (1+1+2+4+4)
     34 
     35 //===----------------------------------------------------------------------===//
     36 // PTHLexer methods.
     37 //===----------------------------------------------------------------------===//
     38 
     39 PTHLexer::PTHLexer(Preprocessor &PP, FileID FID, const unsigned char *D,
     40                    const unsigned char *ppcond, PTHManager &PM)
     41   : PreprocessorLexer(&PP, FID), TokBuf(D), CurPtr(D), LastHashTokPtr(0),
     42     PPCond(ppcond), CurPPCondPtr(ppcond), PTHMgr(PM) {
     43 
     44   FileStartLoc = PP.getSourceManager().getLocForStartOfFile(FID);
     45 }
     46 
     47 void PTHLexer::Lex(Token& Tok) {
     48 LexNextToken:
     49 
     50   //===--------------------------------------==//
     51   // Read the raw token data.
     52   //===--------------------------------------==//
     53 
     54   // Shadow CurPtr into an automatic variable.
     55   const unsigned char *CurPtrShadow = CurPtr;
     56 
     57   // Read in the data for the token.
     58   unsigned Word0 = ReadLE32(CurPtrShadow);
     59   uint32_t IdentifierID = ReadLE32(CurPtrShadow);
     60   uint32_t FileOffset = ReadLE32(CurPtrShadow);
     61 
     62   tok::TokenKind TKind = (tok::TokenKind) (Word0 & 0xFF);
     63   Token::TokenFlags TFlags = (Token::TokenFlags) ((Word0 >> 8) & 0xFF);
     64   uint32_t Len = Word0 >> 16;
     65 
     66   CurPtr = CurPtrShadow;
     67 
     68   //===--------------------------------------==//
     69   // Construct the token itself.
     70   //===--------------------------------------==//
     71 
     72   Tok.startToken();
     73   Tok.setKind(TKind);
     74   Tok.setFlag(TFlags);
     75   assert(!LexingRawMode);
     76   Tok.setLocation(FileStartLoc.getLocWithOffset(FileOffset));
     77   Tok.setLength(Len);
     78 
     79   // Handle identifiers.
     80   if (Tok.isLiteral()) {
     81     Tok.setLiteralData((const char*) (PTHMgr.SpellingBase + IdentifierID));
     82   }
     83   else if (IdentifierID) {
     84     MIOpt.ReadToken();
     85     IdentifierInfo *II = PTHMgr.GetIdentifierInfo(IdentifierID-1);
     86 
     87     Tok.setIdentifierInfo(II);
     88 
     89     // Change the kind of this identifier to the appropriate token kind, e.g.
     90     // turning "for" into a keyword.
     91     Tok.setKind(II->getTokenID());
     92 
     93     if (II->isHandleIdentifierCase())
     94       PP->HandleIdentifier(Tok);
     95     return;
     96   }
     97 
     98   //===--------------------------------------==//
     99   // Process the token.
    100   //===--------------------------------------==//
    101   if (TKind == tok::eof) {
    102     // Save the end-of-file token.
    103     EofToken = Tok;
    104 
    105     // Save 'PP' to 'PPCache' as LexEndOfFile can delete 'this'.
    106     Preprocessor *PPCache = PP;
    107 
    108     assert(!ParsingPreprocessorDirective);
    109     assert(!LexingRawMode);
    110 
    111     if (LexEndOfFile(Tok))
    112       return;
    113 
    114     return PPCache->Lex(Tok);
    115   }
    116 
    117   if (TKind == tok::hash && Tok.isAtStartOfLine()) {
    118     LastHashTokPtr = CurPtr - DISK_TOKEN_SIZE;
    119     assert(!LexingRawMode);
    120     PP->HandleDirective(Tok);
    121 
    122     if (PP->isCurrentLexer(this))
    123       goto LexNextToken;
    124 
    125     return PP->Lex(Tok);
    126   }
    127 
    128   if (TKind == tok::eod) {
    129     assert(ParsingPreprocessorDirective);
    130     ParsingPreprocessorDirective = false;
    131     return;
    132   }
    133 
    134   MIOpt.ReadToken();
    135 }
    136 
    137 bool PTHLexer::LexEndOfFile(Token &Result) {
    138   // If we hit the end of the file while parsing a preprocessor directive,
    139   // end the preprocessor directive first.  The next token returned will
    140   // then be the end of file.
    141   if (ParsingPreprocessorDirective) {
    142     ParsingPreprocessorDirective = false; // Done parsing the "line".
    143     return true;  // Have a token.
    144   }
    145 
    146   assert(!LexingRawMode);
    147 
    148   // If we are in a #if directive, emit an error.
    149   while (!ConditionalStack.empty()) {
    150     if (PP->getCodeCompletionFileLoc() != FileStartLoc)
    151       PP->Diag(ConditionalStack.back().IfLoc,
    152                diag::err_pp_unterminated_conditional);
    153     ConditionalStack.pop_back();
    154   }
    155 
    156   // Finally, let the preprocessor handle this.
    157   return PP->HandleEndOfFile(Result);
    158 }
    159 
    160 // FIXME: We can just grab the last token instead of storing a copy
    161 // into EofToken.
    162 void PTHLexer::getEOF(Token& Tok) {
    163   assert(EofToken.is(tok::eof));
    164   Tok = EofToken;
    165 }
    166 
    167 void PTHLexer::DiscardToEndOfLine() {
    168   assert(ParsingPreprocessorDirective && ParsingFilename == false &&
    169          "Must be in a preprocessing directive!");
    170 
    171   // We assume that if the preprocessor wishes to discard to the end of
    172   // the line that it also means to end the current preprocessor directive.
    173   ParsingPreprocessorDirective = false;
    174 
    175   // Skip tokens by only peeking at their token kind and the flags.
    176   // We don't need to actually reconstruct full tokens from the token buffer.
    177   // This saves some copies and it also reduces IdentifierInfo* lookup.
    178   const unsigned char* p = CurPtr;
    179   while (1) {
    180     // Read the token kind.  Are we at the end of the file?
    181     tok::TokenKind x = (tok::TokenKind) (uint8_t) *p;
    182     if (x == tok::eof) break;
    183 
    184     // Read the token flags.  Are we at the start of the next line?
    185     Token::TokenFlags y = (Token::TokenFlags) (uint8_t) p[1];
    186     if (y & Token::StartOfLine) break;
    187 
    188     // Skip to the next token.
    189     p += DISK_TOKEN_SIZE;
    190   }
    191 
    192   CurPtr = p;
    193 }
    194 
    195 /// SkipBlock - Used by Preprocessor to skip the current conditional block.
    196 bool PTHLexer::SkipBlock() {
    197   assert(CurPPCondPtr && "No cached PP conditional information.");
    198   assert(LastHashTokPtr && "No known '#' token.");
    199 
    200   const unsigned char* HashEntryI = 0;
    201   uint32_t Offset;
    202   uint32_t TableIdx;
    203 
    204   do {
    205     // Read the token offset from the side-table.
    206     Offset = ReadLE32(CurPPCondPtr);
    207 
    208     // Read the target table index from the side-table.
    209     TableIdx = ReadLE32(CurPPCondPtr);
    210 
    211     // Compute the actual memory address of the '#' token data for this entry.
    212     HashEntryI = TokBuf + Offset;
    213 
    214     // Optmization: "Sibling jumping".  #if...#else...#endif blocks can
    215     //  contain nested blocks.  In the side-table we can jump over these
    216     //  nested blocks instead of doing a linear search if the next "sibling"
    217     //  entry is not at a location greater than LastHashTokPtr.
    218     if (HashEntryI < LastHashTokPtr && TableIdx) {
    219       // In the side-table we are still at an entry for a '#' token that
    220       // is earlier than the last one we saw.  Check if the location we would
    221       // stride gets us closer.
    222       const unsigned char* NextPPCondPtr =
    223         PPCond + TableIdx*(sizeof(uint32_t)*2);
    224       assert(NextPPCondPtr >= CurPPCondPtr);
    225       // Read where we should jump to.
    226       uint32_t TmpOffset = ReadLE32(NextPPCondPtr);
    227       const unsigned char* HashEntryJ = TokBuf + TmpOffset;
    228 
    229       if (HashEntryJ <= LastHashTokPtr) {
    230         // Jump directly to the next entry in the side table.
    231         HashEntryI = HashEntryJ;
    232         Offset = TmpOffset;
    233         TableIdx = ReadLE32(NextPPCondPtr);
    234         CurPPCondPtr = NextPPCondPtr;
    235       }
    236     }
    237   }
    238   while (HashEntryI < LastHashTokPtr);
    239   assert(HashEntryI == LastHashTokPtr && "No PP-cond entry found for '#'");
    240   assert(TableIdx && "No jumping from #endifs.");
    241 
    242   // Update our side-table iterator.
    243   const unsigned char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2);
    244   assert(NextPPCondPtr >= CurPPCondPtr);
    245   CurPPCondPtr = NextPPCondPtr;
    246 
    247   // Read where we should jump to.
    248   HashEntryI = TokBuf + ReadLE32(NextPPCondPtr);
    249   uint32_t NextIdx = ReadLE32(NextPPCondPtr);
    250 
    251   // By construction NextIdx will be zero if this is a #endif.  This is useful
    252   // to know to obviate lexing another token.
    253   bool isEndif = NextIdx == 0;
    254 
    255   // This case can occur when we see something like this:
    256   //
    257   //  #if ...
    258   //   /* a comment or nothing */
    259   //  #elif
    260   //
    261   // If we are skipping the first #if block it will be the case that CurPtr
    262   // already points 'elif'.  Just return.
    263 
    264   if (CurPtr > HashEntryI) {
    265     assert(CurPtr == HashEntryI + DISK_TOKEN_SIZE);
    266     // Did we reach a #endif?  If so, go ahead and consume that token as well.
    267     if (isEndif)
    268       CurPtr += DISK_TOKEN_SIZE*2;
    269     else
    270       LastHashTokPtr = HashEntryI;
    271 
    272     return isEndif;
    273   }
    274 
    275   // Otherwise, we need to advance.  Update CurPtr to point to the '#' token.
    276   CurPtr = HashEntryI;
    277 
    278   // Update the location of the last observed '#'.  This is useful if we
    279   // are skipping multiple blocks.
    280   LastHashTokPtr = CurPtr;
    281 
    282   // Skip the '#' token.
    283   assert(((tok::TokenKind)*CurPtr) == tok::hash);
    284   CurPtr += DISK_TOKEN_SIZE;
    285 
    286   // Did we reach a #endif?  If so, go ahead and consume that token as well.
    287   if (isEndif) { CurPtr += DISK_TOKEN_SIZE*2; }
    288 
    289   return isEndif;
    290 }
    291 
    292 SourceLocation PTHLexer::getSourceLocation() {
    293   // getSourceLocation is not on the hot path.  It is used to get the location
    294   // of the next token when transitioning back to this lexer when done
    295   // handling a #included file.  Just read the necessary data from the token
    296   // data buffer to construct the SourceLocation object.
    297   // NOTE: This is a virtual function; hence it is defined out-of-line.
    298   const unsigned char *OffsetPtr = CurPtr + (DISK_TOKEN_SIZE - 4);
    299   uint32_t Offset = ReadLE32(OffsetPtr);
    300   return FileStartLoc.getLocWithOffset(Offset);
    301 }
    302 
    303 //===----------------------------------------------------------------------===//
    304 // PTH file lookup: map from strings to file data.
    305 //===----------------------------------------------------------------------===//
    306 
    307 /// PTHFileLookup - This internal data structure is used by the PTHManager
    308 ///  to map from FileEntry objects managed by FileManager to offsets within
    309 ///  the PTH file.
    310 namespace {
    311 class PTHFileData {
    312   const uint32_t TokenOff;
    313   const uint32_t PPCondOff;
    314 public:
    315   PTHFileData(uint32_t tokenOff, uint32_t ppCondOff)
    316     : TokenOff(tokenOff), PPCondOff(ppCondOff) {}
    317 
    318   uint32_t getTokenOffset() const { return TokenOff; }
    319   uint32_t getPPCondOffset() const { return PPCondOff; }
    320 };
    321 
    322 
    323 class PTHFileLookupCommonTrait {
    324 public:
    325   typedef std::pair<unsigned char, const char*> internal_key_type;
    326 
    327   static unsigned ComputeHash(internal_key_type x) {
    328     return llvm::HashString(x.second);
    329   }
    330 
    331   static std::pair<unsigned, unsigned>
    332   ReadKeyDataLength(const unsigned char*& d) {
    333     unsigned keyLen = (unsigned) ReadUnalignedLE16(d);
    334     unsigned dataLen = (unsigned) *(d++);
    335     return std::make_pair(keyLen, dataLen);
    336   }
    337 
    338   static internal_key_type ReadKey(const unsigned char* d, unsigned) {
    339     unsigned char k = *(d++); // Read the entry kind.
    340     return std::make_pair(k, (const char*) d);
    341   }
    342 };
    343 
    344 class PTHFileLookupTrait : public PTHFileLookupCommonTrait {
    345 public:
    346   typedef const FileEntry* external_key_type;
    347   typedef PTHFileData      data_type;
    348 
    349   static internal_key_type GetInternalKey(const FileEntry* FE) {
    350     return std::make_pair((unsigned char) 0x1, FE->getName());
    351   }
    352 
    353   static bool EqualKey(internal_key_type a, internal_key_type b) {
    354     return a.first == b.first && strcmp(a.second, b.second) == 0;
    355   }
    356 
    357   static PTHFileData ReadData(const internal_key_type& k,
    358                               const unsigned char* d, unsigned) {
    359     assert(k.first == 0x1 && "Only file lookups can match!");
    360     uint32_t x = ::ReadUnalignedLE32(d);
    361     uint32_t y = ::ReadUnalignedLE32(d);
    362     return PTHFileData(x, y);
    363   }
    364 };
    365 
    366 class PTHStringLookupTrait {
    367 public:
    368   typedef uint32_t
    369           data_type;
    370 
    371   typedef const std::pair<const char*, unsigned>
    372           external_key_type;
    373 
    374   typedef external_key_type internal_key_type;
    375 
    376   static bool EqualKey(const internal_key_type& a,
    377                        const internal_key_type& b) {
    378     return (a.second == b.second) ? memcmp(a.first, b.first, a.second) == 0
    379                                   : false;
    380   }
    381 
    382   static unsigned ComputeHash(const internal_key_type& a) {
    383     return llvm::HashString(StringRef(a.first, a.second));
    384   }
    385 
    386   // This hopefully will just get inlined and removed by the optimizer.
    387   static const internal_key_type&
    388   GetInternalKey(const external_key_type& x) { return x; }
    389 
    390   static std::pair<unsigned, unsigned>
    391   ReadKeyDataLength(const unsigned char*& d) {
    392     return std::make_pair((unsigned) ReadUnalignedLE16(d), sizeof(uint32_t));
    393   }
    394 
    395   static std::pair<const char*, unsigned>
    396   ReadKey(const unsigned char* d, unsigned n) {
    397       assert(n >= 2 && d[n-1] == '\0');
    398       return std::make_pair((const char*) d, n-1);
    399     }
    400 
    401   static uint32_t ReadData(const internal_key_type& k, const unsigned char* d,
    402                            unsigned) {
    403     return ::ReadUnalignedLE32(d);
    404   }
    405 };
    406 
    407 } // end anonymous namespace
    408 
    409 typedef OnDiskChainedHashTable<PTHFileLookupTrait>   PTHFileLookup;
    410 typedef OnDiskChainedHashTable<PTHStringLookupTrait> PTHStringIdLookup;
    411 
    412 //===----------------------------------------------------------------------===//
    413 // PTHManager methods.
    414 //===----------------------------------------------------------------------===//
    415 
    416 PTHManager::PTHManager(const llvm::MemoryBuffer* buf, void* fileLookup,
    417                        const unsigned char* idDataTable,
    418                        IdentifierInfo** perIDCache,
    419                        void* stringIdLookup, unsigned numIds,
    420                        const unsigned char* spellingBase,
    421                        const char* originalSourceFile)
    422 : Buf(buf), PerIDCache(perIDCache), FileLookup(fileLookup),
    423   IdDataTable(idDataTable), StringIdLookup(stringIdLookup),
    424   NumIds(numIds), PP(0), SpellingBase(spellingBase),
    425   OriginalSourceFile(originalSourceFile) {}
    426 
    427 PTHManager::~PTHManager() {
    428   delete Buf;
    429   delete (PTHFileLookup*) FileLookup;
    430   delete (PTHStringIdLookup*) StringIdLookup;
    431   free(PerIDCache);
    432 }
    433 
    434 static void InvalidPTH(DiagnosticsEngine &Diags, const char *Msg) {
    435   Diags.Report(Diags.getCustomDiagID(DiagnosticsEngine::Error, Msg));
    436 }
    437 
    438 PTHManager *PTHManager::Create(const std::string &file,
    439                                DiagnosticsEngine &Diags) {
    440   // Memory map the PTH file.
    441   llvm::OwningPtr<llvm::MemoryBuffer> File;
    442 
    443   if (llvm::MemoryBuffer::getFile(file, File)) {
    444     // FIXME: Add ec.message() to this diag.
    445     Diags.Report(diag::err_invalid_pth_file) << file;
    446     return 0;
    447   }
    448 
    449   // Get the buffer ranges and check if there are at least three 32-bit
    450   // words at the end of the file.
    451   const unsigned char *BufBeg = (unsigned char*)File->getBufferStart();
    452   const unsigned char *BufEnd = (unsigned char*)File->getBufferEnd();
    453 
    454   // Check the prologue of the file.
    455   if ((BufEnd - BufBeg) < (signed)(sizeof("cfe-pth") + 3 + 4) ||
    456       memcmp(BufBeg, "cfe-pth", sizeof("cfe-pth") - 1) != 0) {
    457     Diags.Report(diag::err_invalid_pth_file) << file;
    458     return 0;
    459   }
    460 
    461   // Read the PTH version.
    462   const unsigned char *p = BufBeg + (sizeof("cfe-pth") - 1);
    463   unsigned Version = ReadLE32(p);
    464 
    465   if (Version < PTHManager::Version) {
    466     InvalidPTH(Diags,
    467         Version < PTHManager::Version
    468         ? "PTH file uses an older PTH format that is no longer supported"
    469         : "PTH file uses a newer PTH format that cannot be read");
    470     return 0;
    471   }
    472 
    473   // Compute the address of the index table at the end of the PTH file.
    474   const unsigned char *PrologueOffset = p;
    475 
    476   if (PrologueOffset >= BufEnd) {
    477     Diags.Report(diag::err_invalid_pth_file) << file;
    478     return 0;
    479   }
    480 
    481   // Construct the file lookup table.  This will be used for mapping from
    482   // FileEntry*'s to cached tokens.
    483   const unsigned char* FileTableOffset = PrologueOffset + sizeof(uint32_t)*2;
    484   const unsigned char* FileTable = BufBeg + ReadLE32(FileTableOffset);
    485 
    486   if (!(FileTable > BufBeg && FileTable < BufEnd)) {
    487     Diags.Report(diag::err_invalid_pth_file) << file;
    488     return 0; // FIXME: Proper error diagnostic?
    489   }
    490 
    491   llvm::OwningPtr<PTHFileLookup> FL(PTHFileLookup::Create(FileTable, BufBeg));
    492 
    493   // Warn if the PTH file is empty.  We still want to create a PTHManager
    494   // as the PTH could be used with -include-pth.
    495   if (FL->isEmpty())
    496     InvalidPTH(Diags, "PTH file contains no cached source data");
    497 
    498   // Get the location of the table mapping from persistent ids to the
    499   // data needed to reconstruct identifiers.
    500   const unsigned char* IDTableOffset = PrologueOffset + sizeof(uint32_t)*0;
    501   const unsigned char* IData = BufBeg + ReadLE32(IDTableOffset);
    502 
    503   if (!(IData >= BufBeg && IData < BufEnd)) {
    504     Diags.Report(diag::err_invalid_pth_file) << file;
    505     return 0;
    506   }
    507 
    508   // Get the location of the hashtable mapping between strings and
    509   // persistent IDs.
    510   const unsigned char* StringIdTableOffset = PrologueOffset + sizeof(uint32_t)*1;
    511   const unsigned char* StringIdTable = BufBeg + ReadLE32(StringIdTableOffset);
    512   if (!(StringIdTable >= BufBeg && StringIdTable < BufEnd)) {
    513     Diags.Report(diag::err_invalid_pth_file) << file;
    514     return 0;
    515   }
    516 
    517   llvm::OwningPtr<PTHStringIdLookup> SL(PTHStringIdLookup::Create(StringIdTable,
    518                                                                   BufBeg));
    519 
    520   // Get the location of the spelling cache.
    521   const unsigned char* spellingBaseOffset = PrologueOffset + sizeof(uint32_t)*3;
    522   const unsigned char* spellingBase = BufBeg + ReadLE32(spellingBaseOffset);
    523   if (!(spellingBase >= BufBeg && spellingBase < BufEnd)) {
    524     Diags.Report(diag::err_invalid_pth_file) << file;
    525     return 0;
    526   }
    527 
    528   // Get the number of IdentifierInfos and pre-allocate the identifier cache.
    529   uint32_t NumIds = ReadLE32(IData);
    530 
    531   // Pre-allocate the persistent ID -> IdentifierInfo* cache.  We use calloc()
    532   // so that we in the best case only zero out memory once when the OS returns
    533   // us new pages.
    534   IdentifierInfo** PerIDCache = 0;
    535 
    536   if (NumIds) {
    537     PerIDCache = (IdentifierInfo**)calloc(NumIds, sizeof(*PerIDCache));
    538     if (!PerIDCache) {
    539       InvalidPTH(Diags, "Could not allocate memory for processing PTH file");
    540       return 0;
    541     }
    542   }
    543 
    544   // Compute the address of the original source file.
    545   const unsigned char* originalSourceBase = PrologueOffset + sizeof(uint32_t)*4;
    546   unsigned len = ReadUnalignedLE16(originalSourceBase);
    547   if (!len) originalSourceBase = 0;
    548 
    549   // Create the new PTHManager.
    550   return new PTHManager(File.take(), FL.take(), IData, PerIDCache,
    551                         SL.take(), NumIds, spellingBase,
    552                         (const char*) originalSourceBase);
    553 }
    554 
    555 IdentifierInfo* PTHManager::LazilyCreateIdentifierInfo(unsigned PersistentID) {
    556   // Look in the PTH file for the string data for the IdentifierInfo object.
    557   const unsigned char* TableEntry = IdDataTable + sizeof(uint32_t)*PersistentID;
    558   const unsigned char* IDData =
    559     (const unsigned char*)Buf->getBufferStart() + ReadLE32(TableEntry);
    560   assert(IDData < (const unsigned char*)Buf->getBufferEnd());
    561 
    562   // Allocate the object.
    563   std::pair<IdentifierInfo,const unsigned char*> *Mem =
    564     Alloc.Allocate<std::pair<IdentifierInfo,const unsigned char*> >();
    565 
    566   Mem->second = IDData;
    567   assert(IDData[0] != '\0');
    568   IdentifierInfo *II = new ((void*) Mem) IdentifierInfo();
    569 
    570   // Store the new IdentifierInfo in the cache.
    571   PerIDCache[PersistentID] = II;
    572   assert(II->getNameStart() && II->getNameStart()[0] != '\0');
    573   return II;
    574 }
    575 
    576 IdentifierInfo* PTHManager::get(StringRef Name) {
    577   PTHStringIdLookup& SL = *((PTHStringIdLookup*)StringIdLookup);
    578   // Double check our assumption that the last character isn't '\0'.
    579   assert(Name.empty() || Name.back() != '\0');
    580   PTHStringIdLookup::iterator I = SL.find(std::make_pair(Name.data(),
    581                                                          Name.size()));
    582   if (I == SL.end()) // No identifier found?
    583     return 0;
    584 
    585   // Match found.  Return the identifier!
    586   assert(*I > 0);
    587   return GetIdentifierInfo(*I-1);
    588 }
    589 
    590 PTHLexer *PTHManager::CreateLexer(FileID FID) {
    591   const FileEntry *FE = PP->getSourceManager().getFileEntryForID(FID);
    592   if (!FE)
    593     return 0;
    594 
    595   // Lookup the FileEntry object in our file lookup data structure.  It will
    596   // return a variant that indicates whether or not there is an offset within
    597   // the PTH file that contains cached tokens.
    598   PTHFileLookup& PFL = *((PTHFileLookup*)FileLookup);
    599   PTHFileLookup::iterator I = PFL.find(FE);
    600 
    601   if (I == PFL.end()) // No tokens available?
    602     return 0;
    603 
    604   const PTHFileData& FileData = *I;
    605 
    606   const unsigned char *BufStart = (const unsigned char *)Buf->getBufferStart();
    607   // Compute the offset of the token data within the buffer.
    608   const unsigned char* data = BufStart + FileData.getTokenOffset();
    609 
    610   // Get the location of pp-conditional table.
    611   const unsigned char* ppcond = BufStart + FileData.getPPCondOffset();
    612   uint32_t Len = ReadLE32(ppcond);
    613   if (Len == 0) ppcond = 0;
    614 
    615   assert(PP && "No preprocessor set yet!");
    616   return new PTHLexer(*PP, FID, data, ppcond, *this);
    617 }
    618 
    619 //===----------------------------------------------------------------------===//
    620 // 'stat' caching.
    621 //===----------------------------------------------------------------------===//
    622 
    623 namespace {
    624 class PTHStatData {
    625 public:
    626   const bool hasStat;
    627   const ino_t ino;
    628   const dev_t dev;
    629   const mode_t mode;
    630   const time_t mtime;
    631   const off_t size;
    632 
    633   PTHStatData(ino_t i, dev_t d, mode_t mo, time_t m, off_t s)
    634   : hasStat(true), ino(i), dev(d), mode(mo), mtime(m), size(s) {}
    635 
    636   PTHStatData()
    637     : hasStat(false), ino(0), dev(0), mode(0), mtime(0), size(0) {}
    638 };
    639 
    640 class PTHStatLookupTrait : public PTHFileLookupCommonTrait {
    641 public:
    642   typedef const char* external_key_type;  // const char*
    643   typedef PTHStatData data_type;
    644 
    645   static internal_key_type GetInternalKey(const char *path) {
    646     // The key 'kind' doesn't matter here because it is ignored in EqualKey.
    647     return std::make_pair((unsigned char) 0x0, path);
    648   }
    649 
    650   static bool EqualKey(internal_key_type a, internal_key_type b) {
    651     // When doing 'stat' lookups we don't care about the kind of 'a' and 'b',
    652     // just the paths.
    653     return strcmp(a.second, b.second) == 0;
    654   }
    655 
    656   static data_type ReadData(const internal_key_type& k, const unsigned char* d,
    657                             unsigned) {
    658 
    659     if (k.first /* File or Directory */) {
    660       if (k.first == 0x1 /* File */) d += 4 * 2; // Skip the first 2 words.
    661       ino_t ino = (ino_t) ReadUnalignedLE32(d);
    662       dev_t dev = (dev_t) ReadUnalignedLE32(d);
    663       mode_t mode = (mode_t) ReadUnalignedLE16(d);
    664       time_t mtime = (time_t) ReadUnalignedLE64(d);
    665       return data_type(ino, dev, mode, mtime, (off_t) ReadUnalignedLE64(d));
    666     }
    667 
    668     // Negative stat.  Don't read anything.
    669     return data_type();
    670   }
    671 };
    672 
    673 class PTHStatCache : public FileSystemStatCache {
    674   typedef OnDiskChainedHashTable<PTHStatLookupTrait> CacheTy;
    675   CacheTy Cache;
    676 
    677 public:
    678   PTHStatCache(PTHFileLookup &FL) :
    679     Cache(FL.getNumBuckets(), FL.getNumEntries(), FL.getBuckets(),
    680           FL.getBase()) {}
    681 
    682   ~PTHStatCache() {}
    683 
    684   LookupResult getStat(const char *Path, struct stat &StatBuf,
    685                        int *FileDescriptor) {
    686     // Do the lookup for the file's data in the PTH file.
    687     CacheTy::iterator I = Cache.find(Path);
    688 
    689     // If we don't get a hit in the PTH file just forward to 'stat'.
    690     if (I == Cache.end())
    691       return statChained(Path, StatBuf, FileDescriptor);
    692 
    693     const PTHStatData &Data = *I;
    694 
    695     if (!Data.hasStat)
    696       return CacheMissing;
    697 
    698     StatBuf.st_ino = Data.ino;
    699     StatBuf.st_dev = Data.dev;
    700     StatBuf.st_mtime = Data.mtime;
    701     StatBuf.st_mode = Data.mode;
    702     StatBuf.st_size = Data.size;
    703     return CacheExists;
    704   }
    705 };
    706 } // end anonymous namespace
    707 
    708 FileSystemStatCache *PTHManager::createStatCache() {
    709   return new PTHStatCache(*((PTHFileLookup*) FileLookup));
    710 }
    711