Home | History | Annotate | Download | only in Serialization 
      1 //===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 // This file implements the GlobalModuleIndex class.
     11 //
     12 //===----------------------------------------------------------------------===//
     13 
     14 #include "ASTReaderInternals.h"
     15 #include "clang/Basic/FileManager.h"
     16 #include "clang/Lex/HeaderSearch.h"
     17 #include "clang/Serialization/ASTBitCodes.h"
     18 #include "clang/Serialization/GlobalModuleIndex.h"
     19 #include "clang/Serialization/Module.h"
     20 #include "llvm/ADT/DenseMap.h"
     21 #include "llvm/ADT/MapVector.h"
     22 #include "llvm/ADT/SmallString.h"
     23 #include "llvm/ADT/StringExtras.h"
     24 #include "llvm/Bitcode/BitstreamReader.h"
     25 #include "llvm/Bitcode/BitstreamWriter.h"
     26 #include "llvm/Support/FileSystem.h"
     27 #include "llvm/Support/LockFileManager.h"
     28 #include "llvm/Support/MemoryBuffer.h"
     29 #include "llvm/Support/OnDiskHashTable.h"
     30 #include "llvm/Support/Path.h"
     31 #include <cstdio>
     32 using namespace clang;
     33 using namespace serialization;
     34 
     35 //----------------------------------------------------------------------------//
     36 // Shared constants
     37 //----------------------------------------------------------------------------//
     38 namespace {
     39   enum {
     40     /// \brief The block containing the index.
     41     GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
     42   };
     43 
     44   /// \brief Describes the record types in the index.
     45   enum IndexRecordTypes {
     46     /// \brief Contains version information and potentially other metadata,
     47     /// used to determine if we can read this global index file.
     48     INDEX_METADATA,
     49     /// \brief Describes a module, including its file name and dependencies.
     50     MODULE,
     51     /// \brief The index for identifiers.
     52     IDENTIFIER_INDEX
     53   };
     54 }
     55 
     56 /// \brief The name of the global index file.
     57 static const char * const IndexFileName = "modules.idx";
     58 
     59 /// \brief The global index file version.
     60 static const unsigned CurrentVersion = 1;
     61 
     62 //----------------------------------------------------------------------------//
     63 // Global module index reader.
     64 //----------------------------------------------------------------------------//
     65 
     66 namespace {
     67 
     68 /// \brief Trait used to read the identifier index from the on-disk hash
     69 /// table.
     70 class IdentifierIndexReaderTrait {
     71 public:
     72   typedef StringRef external_key_type;
     73   typedef StringRef internal_key_type;
     74   typedef SmallVector<unsigned, 2> data_type;
     75   typedef unsigned hash_value_type;
     76   typedef unsigned offset_type;
     77 
     78   static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
     79     return a == b;
     80   }
     81 
     82   static hash_value_type ComputeHash(const internal_key_type& a) {
     83     return llvm::HashString(a);
     84   }
     85 
     86   static std::pair<unsigned, unsigned>
     87   ReadKeyDataLength(const unsigned char*& d) {
     88     using namespace llvm::support;
     89     unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
     90     unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
     91     return std::make_pair(KeyLen, DataLen);
     92   }
     93 
     94   static const internal_key_type&
     95   GetInternalKey(const external_key_type& x) { return x; }
     96 
     97   static const external_key_type&
     98   GetExternalKey(const internal_key_type& x) { return x; }
     99 
    100   static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
    101     return StringRef((const char *)d, n);
    102   }
    103 
    104   static data_type ReadData(const internal_key_type& k,
    105                             const unsigned char* d,
    106                             unsigned DataLen) {
    107     using namespace llvm::support;
    108 
    109     data_type Result;
    110     while (DataLen > 0) {
    111       unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
    112       Result.push_back(ID);
    113       DataLen -= 4;
    114     }
    115 
    116     return Result;
    117   }
    118 };
    119 
    120 typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait>
    121     IdentifierIndexTable;
    122 
    123 }
    124 
    125 GlobalModuleIndex::GlobalModuleIndex(std::unique_ptr<llvm::MemoryBuffer> Buffer,
    126                                      llvm::BitstreamCursor Cursor)
    127     : Buffer(std::move(Buffer)), IdentifierIndex(), NumIdentifierLookups(),
    128       NumIdentifierLookupHits() {
    129   // Read the global index.
    130   bool InGlobalIndexBlock = false;
    131   bool Done = false;
    132   while (!Done) {
    133     llvm::BitstreamEntry Entry = Cursor.advance();
    134 
    135     switch (Entry.Kind) {
    136     case llvm::BitstreamEntry::Error:
    137       return;
    138 
    139     case llvm::BitstreamEntry::EndBlock:
    140       if (InGlobalIndexBlock) {
    141         InGlobalIndexBlock = false;
    142         Done = true;
    143         continue;
    144       }
    145       return;
    146 
    147 
    148     case llvm::BitstreamEntry::Record:
    149       // Entries in the global index block are handled below.
    150       if (InGlobalIndexBlock)
    151         break;
    152 
    153       return;
    154 
    155     case llvm::BitstreamEntry::SubBlock:
    156       if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
    157         if (Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
    158           return;
    159 
    160         InGlobalIndexBlock = true;
    161       } else if (Cursor.SkipBlock()) {
    162         return;
    163       }
    164       continue;
    165     }
    166 
    167     SmallVector<uint64_t, 64> Record;
    168     StringRef Blob;
    169     switch ((IndexRecordTypes)Cursor.readRecord(Entry.ID, Record, &Blob)) {
    170     case INDEX_METADATA:
    171       // Make sure that the version matches.
    172       if (Record.size() < 1 || Record[0] != CurrentVersion)
    173         return;
    174       break;
    175 
    176     case MODULE: {
    177       unsigned Idx = 0;
    178       unsigned ID = Record[Idx++];
    179 
    180       // Make room for this module's information.
    181       if (ID == Modules.size())
    182         Modules.push_back(ModuleInfo());
    183       else
    184         Modules.resize(ID + 1);
    185 
    186       // Size/modification time for this module file at the time the
    187       // global index was built.
    188       Modules[ID].Size = Record[Idx++];
    189       Modules[ID].ModTime = Record[Idx++];
    190 
    191       // File name.
    192       unsigned NameLen = Record[Idx++];
    193       Modules[ID].FileName.assign(Record.begin() + Idx,
    194                                   Record.begin() + Idx + NameLen);
    195       Idx += NameLen;
    196 
    197       // Dependencies
    198       unsigned NumDeps = Record[Idx++];
    199       Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(),
    200                                       Record.begin() + Idx,
    201                                       Record.begin() + Idx + NumDeps);
    202       Idx += NumDeps;
    203 
    204       // Make sure we're at the end of the record.
    205       assert(Idx == Record.size() && "More module info?");
    206 
    207       // Record this module as an unresolved module.
    208       // FIXME: this doesn't work correctly for module names containing path
    209       // separators.
    210       StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName);
    211       // Remove the -<hash of ModuleMapPath>
    212       ModuleName = ModuleName.rsplit('-').first;
    213       UnresolvedModules[ModuleName] = ID;
    214       break;
    215     }
    216 
    217     case IDENTIFIER_INDEX:
    218       // Wire up the identifier index.
    219       if (Record[0]) {
    220         IdentifierIndex = IdentifierIndexTable::Create(
    221             (const unsigned char *)Blob.data() + Record[0],
    222             (const unsigned char *)Blob.data() + sizeof(uint32_t),
    223             (const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
    224       }
    225       break;
    226     }
    227   }
    228 }
    229 
    230 GlobalModuleIndex::~GlobalModuleIndex() {
    231   delete static_cast<IdentifierIndexTable *>(IdentifierIndex);
    232 }
    233 
    234 std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode>
    235 GlobalModuleIndex::readIndex(StringRef Path) {
    236   // Load the index file, if it's there.
    237   llvm::SmallString<128> IndexPath;
    238   IndexPath += Path;
    239   llvm::sys::path::append(IndexPath, IndexFileName);
    240 
    241   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
    242       llvm::MemoryBuffer::getFile(IndexPath.c_str());
    243   if (!BufferOrErr)
    244     return std::make_pair(nullptr, EC_NotFound);
    245   std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
    246 
    247   /// \brief The bitstream reader from which we'll read the AST file.
    248   llvm::BitstreamReader Reader((const unsigned char *)Buffer->getBufferStart(),
    249                                (const unsigned char *)Buffer->getBufferEnd());
    250 
    251   /// \brief The main bitstream cursor for the main block.
    252   llvm::BitstreamCursor Cursor(Reader);
    253 
    254   // Sniff for the signature.
    255   if (Cursor.Read(8) != 'B' ||
    256       Cursor.Read(8) != 'C' ||
    257       Cursor.Read(8) != 'G' ||
    258       Cursor.Read(8) != 'I') {
    259     return std::make_pair(nullptr, EC_IOError);
    260   }
    261 
    262   return std::make_pair(new GlobalModuleIndex(std::move(Buffer), Cursor),
    263                         EC_None);
    264 }
    265 
    266 void
    267 GlobalModuleIndex::getKnownModules(SmallVectorImpl<ModuleFile *> &ModuleFiles) {
    268   ModuleFiles.clear();
    269   for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
    270     if (ModuleFile *MF = Modules[I].File)
    271       ModuleFiles.push_back(MF);
    272   }
    273 }
    274 
    275 void GlobalModuleIndex::getModuleDependencies(
    276        ModuleFile *File,
    277        SmallVectorImpl<ModuleFile *> &Dependencies) {
    278   // Look for information about this module file.
    279   llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
    280     = ModulesByFile.find(File);
    281   if (Known == ModulesByFile.end())
    282     return;
    283 
    284   // Record dependencies.
    285   Dependencies.clear();
    286   ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
    287   for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
    288     if (ModuleFile *MF = Modules[I].File)
    289       Dependencies.push_back(MF);
    290   }
    291 }
    292 
    293 bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
    294   Hits.clear();
    295 
    296   // If there's no identifier index, there is nothing we can do.
    297   if (!IdentifierIndex)
    298     return false;
    299 
    300   // Look into the identifier index.
    301   ++NumIdentifierLookups;
    302   IdentifierIndexTable &Table
    303     = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
    304   IdentifierIndexTable::iterator Known = Table.find(Name);
    305   if (Known == Table.end()) {
    306     return true;
    307   }
    308 
    309   SmallVector<unsigned, 2> ModuleIDs = *Known;
    310   for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
    311     if (ModuleFile *MF = Modules[ModuleIDs[I]].File)
    312       Hits.insert(MF);
    313   }
    314 
    315   ++NumIdentifierLookupHits;
    316   return true;
    317 }
    318 
    319 bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) {
    320   // Look for the module in the global module index based on the module name.
    321   StringRef Name = File->ModuleName;
    322   llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
    323   if (Known == UnresolvedModules.end()) {
    324     return true;
    325   }
    326 
    327   // Rectify this module with the global module index.
    328   ModuleInfo &Info = Modules[Known->second];
    329 
    330   //  If the size and modification time match what we expected, record this
    331   // module file.
    332   bool Failed = true;
    333   if (File->File->getSize() == Info.Size &&
    334       File->File->getModificationTime() == Info.ModTime) {
    335     Info.File = File;
    336     ModulesByFile[File] = Known->second;
    337 
    338     Failed = false;
    339   }
    340 
    341   // One way or another, we have resolved this module file.
    342   UnresolvedModules.erase(Known);
    343   return Failed;
    344 }
    345 
    346 void GlobalModuleIndex::printStats() {
    347   std::fprintf(stderr, "*** Global Module Index Statistics:\n");
    348   if (NumIdentifierLookups) {
    349     fprintf(stderr, "  %u / %u identifier lookups succeeded (%f%%)\n",
    350             NumIdentifierLookupHits, NumIdentifierLookups,
    351             (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
    352   }
    353   std::fprintf(stderr, "\n");
    354 }
    355 
    356 void GlobalModuleIndex::dump() {
    357   llvm::errs() << "*** Global Module Index Dump:\n";
    358   llvm::errs() << "Module files:\n";
    359   for (auto &MI : Modules) {
    360     llvm::errs() << "** " << MI.FileName << "\n";
    361     if (MI.File)
    362       MI.File->dump();
    363     else
    364       llvm::errs() << "\n";
    365   }
    366   llvm::errs() << "\n";
    367 }
    368 
    369 //----------------------------------------------------------------------------//
    370 // Global module index writer.
    371 //----------------------------------------------------------------------------//
    372 
    373 namespace {
    374   /// \brief Provides information about a specific module file.
    375   struct ModuleFileInfo {
    376     /// \brief The numberic ID for this module file.
    377     unsigned ID;
    378 
    379     /// \brief The set of modules on which this module depends. Each entry is
    380     /// a module ID.
    381     SmallVector<unsigned, 4> Dependencies;
    382   };
    383 
    384   /// \brief Builder that generates the global module index file.
    385   class GlobalModuleIndexBuilder {
    386     FileManager &FileMgr;
    387 
    388     /// \brief Mapping from files to module file information.
    389     typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
    390 
    391     /// \brief Information about each of the known module files.
    392     ModuleFilesMap ModuleFiles;
    393 
    394     /// \brief Mapping from identifiers to the list of module file IDs that
    395     /// consider this identifier to be interesting.
    396     typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
    397 
    398     /// \brief A mapping from all interesting identifiers to the set of module
    399     /// files in which those identifiers are considered interesting.
    400     InterestingIdentifierMap InterestingIdentifiers;
    401 
    402     /// \brief Write the block-info block for the global module index file.
    403     void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
    404 
    405     /// \brief Retrieve the module file information for the given file.
    406     ModuleFileInfo &getModuleFileInfo(const FileEntry *File) {
    407       llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
    408         = ModuleFiles.find(File);
    409       if (Known != ModuleFiles.end())
    410         return Known->second;
    411 
    412       unsigned NewID = ModuleFiles.size();
    413       ModuleFileInfo &Info = ModuleFiles[File];
    414       Info.ID = NewID;
    415       return Info;
    416     }
    417 
    418   public:
    419     explicit GlobalModuleIndexBuilder(FileManager &FileMgr) : FileMgr(FileMgr){}
    420 
    421     /// \brief Load the contents of the given module file into the builder.
    422     ///
    423     /// \returns true if an error occurred, false otherwise.
    424     bool loadModuleFile(const FileEntry *File);
    425 
    426     /// \brief Write the index to the given bitstream.
    427     void writeIndex(llvm::BitstreamWriter &Stream);
    428   };
    429 }
    430 
    431 static void emitBlockID(unsigned ID, const char *Name,
    432                         llvm::BitstreamWriter &Stream,
    433                         SmallVectorImpl<uint64_t> &Record) {
    434   Record.clear();
    435   Record.push_back(ID);
    436   Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
    437 
    438   // Emit the block name if present.
    439   if (!Name || Name[0] == 0) return;
    440   Record.clear();
    441   while (*Name)
    442     Record.push_back(*Name++);
    443   Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
    444 }
    445 
    446 static void emitRecordID(unsigned ID, const char *Name,
    447                          llvm::BitstreamWriter &Stream,
    448                          SmallVectorImpl<uint64_t> &Record) {
    449   Record.clear();
    450   Record.push_back(ID);
    451   while (*Name)
    452     Record.push_back(*Name++);
    453   Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
    454 }
    455 
    456 void
    457 GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
    458   SmallVector<uint64_t, 64> Record;
    459   Stream.EnterSubblock(llvm::bitc::BLOCKINFO_BLOCK_ID, 3);
    460 
    461 #define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
    462 #define RECORD(X) emitRecordID(X, #X, Stream, Record)
    463   BLOCK(GLOBAL_INDEX_BLOCK);
    464   RECORD(INDEX_METADATA);
    465   RECORD(MODULE);
    466   RECORD(IDENTIFIER_INDEX);
    467 #undef RECORD
    468 #undef BLOCK
    469 
    470   Stream.ExitBlock();
    471 }
    472 
    473 namespace {
    474   class InterestingASTIdentifierLookupTrait
    475     : public serialization::reader::ASTIdentifierLookupTraitBase {
    476 
    477   public:
    478     /// \brief The identifier and whether it is "interesting".
    479     typedef std::pair<StringRef, bool> data_type;
    480 
    481     data_type ReadData(const internal_key_type& k,
    482                        const unsigned char* d,
    483                        unsigned DataLen) {
    484       // The first bit indicates whether this identifier is interesting.
    485       // That's all we care about.
    486       using namespace llvm::support;
    487       unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
    488       bool IsInteresting = RawID & 0x01;
    489       return std::make_pair(k, IsInteresting);
    490     }
    491   };
    492 }
    493 
    494 bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
    495   // Open the module file.
    496 
    497   auto Buffer = FileMgr.getBufferForFile(File, /*isVolatile=*/true);
    498   if (!Buffer) {
    499     return true;
    500   }
    501 
    502   // Initialize the input stream
    503   llvm::BitstreamReader InStreamFile;
    504   InStreamFile.init((const unsigned char *)(*Buffer)->getBufferStart(),
    505                     (const unsigned char *)(*Buffer)->getBufferEnd());
    506   llvm::BitstreamCursor InStream(InStreamFile);
    507 
    508   // Sniff for the signature.
    509   if (InStream.Read(8) != 'C' ||
    510       InStream.Read(8) != 'P' ||
    511       InStream.Read(8) != 'C' ||
    512       InStream.Read(8) != 'H') {
    513     return true;
    514   }
    515 
    516   // Record this module file and assign it a unique ID (if it doesn't have
    517   // one already).
    518   unsigned ID = getModuleFileInfo(File).ID;
    519 
    520   // Search for the blocks and records we care about.
    521   enum { Other, ControlBlock, ASTBlock } State = Other;
    522   bool Done = false;
    523   while (!Done) {
    524     llvm::BitstreamEntry Entry = InStream.advance();
    525     switch (Entry.Kind) {
    526     case llvm::BitstreamEntry::Error:
    527       Done = true;
    528       continue;
    529 
    530     case llvm::BitstreamEntry::Record:
    531       // In the 'other' state, just skip the record. We don't care.
    532       if (State == Other) {
    533         InStream.skipRecord(Entry.ID);
    534         continue;
    535       }
    536 
    537       // Handle potentially-interesting records below.
    538       break;
    539 
    540     case llvm::BitstreamEntry::SubBlock:
    541       if (Entry.ID == CONTROL_BLOCK_ID) {
    542         if (InStream.EnterSubBlock(CONTROL_BLOCK_ID))
    543           return true;
    544 
    545         // Found the control block.
    546         State = ControlBlock;
    547         continue;
    548       }
    549 
    550       if (Entry.ID == AST_BLOCK_ID) {
    551         if (InStream.EnterSubBlock(AST_BLOCK_ID))
    552           return true;
    553 
    554         // Found the AST block.
    555         State = ASTBlock;
    556         continue;
    557       }
    558 
    559       if (InStream.SkipBlock())
    560         return true;
    561 
    562       continue;
    563 
    564     case llvm::BitstreamEntry::EndBlock:
    565       State = Other;
    566       continue;
    567     }
    568 
    569     // Read the given record.
    570     SmallVector<uint64_t, 64> Record;
    571     StringRef Blob;
    572     unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob);
    573 
    574     // Handle module dependencies.
    575     if (State == ControlBlock && Code == IMPORTS) {
    576       // Load each of the imported PCH files.
    577       unsigned Idx = 0, N = Record.size();
    578       while (Idx < N) {
    579         // Read information about the AST file.
    580 
    581         // Skip the imported kind
    582         ++Idx;
    583 
    584         // Skip the import location
    585         ++Idx;
    586 
    587         // Load stored size/modification time.
    588         off_t StoredSize = (off_t)Record[Idx++];
    589         time_t StoredModTime = (time_t)Record[Idx++];
    590 
    591         // Skip the stored signature.
    592         // FIXME: we could read the signature out of the import and validate it.
    593         Idx++;
    594 
    595         // Retrieve the imported file name.
    596         unsigned Length = Record[Idx++];
    597         SmallString<128> ImportedFile(Record.begin() + Idx,
    598                                       Record.begin() + Idx + Length);
    599         Idx += Length;
    600 
    601         // Find the imported module file.
    602         const FileEntry *DependsOnFile
    603           = FileMgr.getFile(ImportedFile, /*openFile=*/false,
    604                             /*cacheFailure=*/false);
    605         if (!DependsOnFile ||
    606             (StoredSize != DependsOnFile->getSize()) ||
    607             (StoredModTime != DependsOnFile->getModificationTime()))
    608           return true;
    609 
    610         // Record the dependency.
    611         unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
    612         getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
    613       }
    614 
    615       continue;
    616     }
    617 
    618     // Handle the identifier table
    619     if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) {
    620       typedef llvm::OnDiskIterableChainedHashTable<
    621           InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
    622       std::unique_ptr<InterestingIdentifierTable> Table(
    623           InterestingIdentifierTable::Create(
    624               (const unsigned char *)Blob.data() + Record[0],
    625               (const unsigned char *)Blob.data() + sizeof(uint32_t),
    626               (const unsigned char *)Blob.data()));
    627       for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
    628                                                      DEnd = Table->data_end();
    629            D != DEnd; ++D) {
    630         std::pair<StringRef, bool> Ident = *D;
    631         if (Ident.second)
    632           InterestingIdentifiers[Ident.first].push_back(ID);
    633         else
    634           (void)InterestingIdentifiers[Ident.first];
    635       }
    636     }
    637 
    638     // We don't care about this record.
    639   }
    640 
    641   return false;
    642 }
    643 
    644 namespace {
    645 
    646 /// \brief Trait used to generate the identifier index as an on-disk hash
    647 /// table.
    648 class IdentifierIndexWriterTrait {
    649 public:
    650   typedef StringRef key_type;
    651   typedef StringRef key_type_ref;
    652   typedef SmallVector<unsigned, 2> data_type;
    653   typedef const SmallVector<unsigned, 2> &data_type_ref;
    654   typedef unsigned hash_value_type;
    655   typedef unsigned offset_type;
    656 
    657   static hash_value_type ComputeHash(key_type_ref Key) {
    658     return llvm::HashString(Key);
    659   }
    660 
    661   std::pair<unsigned,unsigned>
    662   EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
    663     using namespace llvm::support;
    664     endian::Writer<little> LE(Out);
    665     unsigned KeyLen = Key.size();
    666     unsigned DataLen = Data.size() * 4;
    667     LE.write<uint16_t>(KeyLen);
    668     LE.write<uint16_t>(DataLen);
    669     return std::make_pair(KeyLen, DataLen);
    670   }
    671 
    672   void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
    673     Out.write(Key.data(), KeyLen);
    674   }
    675 
    676   void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
    677                 unsigned DataLen) {
    678     using namespace llvm::support;
    679     for (unsigned I = 0, N = Data.size(); I != N; ++I)
    680       endian::Writer<little>(Out).write<uint32_t>(Data[I]);
    681   }
    682 };
    683 
    684 }
    685 
    686 void GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
    687   using namespace llvm;
    688 
    689   // Emit the file header.
    690   Stream.Emit((unsigned)'B', 8);
    691   Stream.Emit((unsigned)'C', 8);
    692   Stream.Emit((unsigned)'G', 8);
    693   Stream.Emit((unsigned)'I', 8);
    694 
    695   // Write the block-info block, which describes the records in this bitcode
    696   // file.
    697   emitBlockInfoBlock(Stream);
    698 
    699   Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
    700 
    701   // Write the metadata.
    702   SmallVector<uint64_t, 2> Record;
    703   Record.push_back(CurrentVersion);
    704   Stream.EmitRecord(INDEX_METADATA, Record);
    705 
    706   // Write the set of known module files.
    707   for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
    708                                 MEnd = ModuleFiles.end();
    709        M != MEnd; ++M) {
    710     Record.clear();
    711     Record.push_back(M->second.ID);
    712     Record.push_back(M->first->getSize());
    713     Record.push_back(M->first->getModificationTime());
    714 
    715     // File name
    716     StringRef Name(M->first->getName());
    717     Record.push_back(Name.size());
    718     Record.append(Name.begin(), Name.end());
    719 
    720     // Dependencies
    721     Record.push_back(M->second.Dependencies.size());
    722     Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
    723     Stream.EmitRecord(MODULE, Record);
    724   }
    725 
    726   // Write the identifier -> module file mapping.
    727   {
    728     llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
    729     IdentifierIndexWriterTrait Trait;
    730 
    731     // Populate the hash table.
    732     for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
    733                                             IEnd = InterestingIdentifiers.end();
    734          I != IEnd; ++I) {
    735       Generator.insert(I->first(), I->second, Trait);
    736     }
    737 
    738     // Create the on-disk hash table in a buffer.
    739     SmallString<4096> IdentifierTable;
    740     uint32_t BucketOffset;
    741     {
    742       using namespace llvm::support;
    743       llvm::raw_svector_ostream Out(IdentifierTable);
    744       // Make sure that no bucket is at offset 0
    745       endian::Writer<little>(Out).write<uint32_t>(0);
    746       BucketOffset = Generator.Emit(Out, Trait);
    747     }
    748 
    749     // Create a blob abbreviation
    750     BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
    751     Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
    752     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
    753     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
    754     unsigned IDTableAbbrev = Stream.EmitAbbrev(Abbrev);
    755 
    756     // Write the identifier table
    757     Record.clear();
    758     Record.push_back(IDENTIFIER_INDEX);
    759     Record.push_back(BucketOffset);
    760     Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable);
    761   }
    762 
    763   Stream.ExitBlock();
    764 }
    765 
    766 GlobalModuleIndex::ErrorCode
    767 GlobalModuleIndex::writeIndex(FileManager &FileMgr, StringRef Path) {
    768   llvm::SmallString<128> IndexPath;
    769   IndexPath += Path;
    770   llvm::sys::path::append(IndexPath, IndexFileName);
    771 
    772   // Coordinate building the global index file with other processes that might
    773   // try to do the same.
    774   llvm::LockFileManager Locked(IndexPath);
    775   switch (Locked) {
    776   case llvm::LockFileManager::LFS_Error:
    777     return EC_IOError;
    778 
    779   case llvm::LockFileManager::LFS_Owned:
    780     // We're responsible for building the index ourselves. Do so below.
    781     break;
    782 
    783   case llvm::LockFileManager::LFS_Shared:
    784     // Someone else is responsible for building the index. We don't care
    785     // when they finish, so we're done.
    786     return EC_Building;
    787   }
    788 
    789   // The module index builder.
    790   GlobalModuleIndexBuilder Builder(FileMgr);
    791 
    792   // Load each of the module files.
    793   std::error_code EC;
    794   for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
    795        D != DEnd && !EC;
    796        D.increment(EC)) {
    797     // If this isn't a module file, we don't care.
    798     if (llvm::sys::path::extension(D->path()) != ".pcm") {
    799       // ... unless it's a .pcm.lock file, which indicates that someone is
    800       // in the process of rebuilding a module. They'll rebuild the index
    801       // at the end of that translation unit, so we don't have to.
    802       if (llvm::sys::path::extension(D->path()) == ".pcm.lock")
    803         return EC_Building;
    804 
    805       continue;
    806     }
    807 
    808     // If we can't find the module file, skip it.
    809     const FileEntry *ModuleFile = FileMgr.getFile(D->path());
    810     if (!ModuleFile)
    811       continue;
    812 
    813     // Load this module file.
    814     if (Builder.loadModuleFile(ModuleFile))
    815       return EC_IOError;
    816   }
    817 
    818   // The output buffer, into which the global index will be written.
    819   SmallVector<char, 16> OutputBuffer;
    820   {
    821     llvm::BitstreamWriter OutputStream(OutputBuffer);
    822     Builder.writeIndex(OutputStream);
    823   }
    824 
    825   // Write the global index file to a temporary file.
    826   llvm::SmallString<128> IndexTmpPath;
    827   int TmpFD;
    828   if (llvm::sys::fs::createUniqueFile(IndexPath + "-%%%%%%%%", TmpFD,
    829                                       IndexTmpPath))
    830     return EC_IOError;
    831 
    832   // Open the temporary global index file for output.
    833   llvm::raw_fd_ostream Out(TmpFD, true);
    834   if (Out.has_error())
    835     return EC_IOError;
    836 
    837   // Write the index.
    838   Out.write(OutputBuffer.data(), OutputBuffer.size());
    839   Out.close();
    840   if (Out.has_error())
    841     return EC_IOError;
    842 
    843   // Remove the old index file. It isn't relevant any more.
    844   llvm::sys::fs::remove(IndexPath);
    845 
    846   // Rename the newly-written index file to the proper name.
    847   if (llvm::sys::fs::rename(IndexTmpPath, IndexPath)) {
    848     // Rename failed; just remove the
    849     llvm::sys::fs::remove(IndexTmpPath);
    850     return EC_IOError;
    851   }
    852 
    853   // We're done.
    854   return EC_None;
    855 }
    856 
    857 namespace {
    858   class GlobalIndexIdentifierIterator : public IdentifierIterator {
    859     /// \brief The current position within the identifier lookup table.
    860     IdentifierIndexTable::key_iterator Current;
    861 
    862     /// \brief The end position within the identifier lookup table.
    863     IdentifierIndexTable::key_iterator End;
    864 
    865   public:
    866     explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
    867       Current = Idx.key_begin();
    868       End = Idx.key_end();
    869     }
    870 
    871     StringRef Next() override {
    872       if (Current == End)
    873         return StringRef();
    874 
    875       StringRef Result = *Current;
    876       ++Current;
    877       return Result;
    878     }
    879   };
    880 }
    881 
    882 IdentifierIterator *GlobalModuleIndex::createIdentifierIterator() const {
    883   IdentifierIndexTable &Table =
    884     *static_cast<IdentifierIndexTable *>(IdentifierIndex);
    885   return new GlobalIndexIdentifierIterator(Table);
    886 }
    887