Home | History | Annotate | Download | only in Serialization 
      1 //===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 // This file implements the GlobalModuleIndex class.
     11 //
     12 //===----------------------------------------------------------------------===//
     13 
     14 #include "ASTReaderInternals.h"
     15 #include "clang/Basic/FileManager.h"
     16 #include "clang/Basic/OnDiskHashTable.h"
     17 #include "clang/Serialization/ASTBitCodes.h"
     18 #include "clang/Serialization/GlobalModuleIndex.h"
     19 #include "clang/Serialization/Module.h"
     20 #include "llvm/ADT/DenseMap.h"
     21 #include "llvm/ADT/MapVector.h"
     22 #include "llvm/ADT/SmallString.h"
     23 #include "llvm/ADT/StringExtras.h"
     24 #include "llvm/Bitcode/BitstreamReader.h"
     25 #include "llvm/Bitcode/BitstreamWriter.h"
     26 #include "llvm/Support/FileSystem.h"
     27 #include "llvm/Support/LockFileManager.h"
     28 #include "llvm/Support/MemoryBuffer.h"
     29 #include "llvm/Support/Path.h"
     30 #include <cstdio>
     31 using namespace clang;
     32 using namespace serialization;
     33 
     34 //----------------------------------------------------------------------------//
     35 // Shared constants
     36 //----------------------------------------------------------------------------//
     37 namespace {
     38   enum {
     39     /// \brief The block containing the index.
     40     GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
     41   };
     42 
     43   /// \brief Describes the record types in the index.
     44   enum IndexRecordTypes {
     45     /// \brief Contains version information and potentially other metadata,
     46     /// used to determine if we can read this global index file.
     47     INDEX_METADATA,
     48     /// \brief Describes a module, including its file name and dependencies.
     49     MODULE,
     50     /// \brief The index for identifiers.
     51     IDENTIFIER_INDEX
     52   };
     53 }
     54 
     55 /// \brief The name of the global index file.
     56 static const char * const IndexFileName = "modules.idx";
     57 
     58 /// \brief The global index file version.
     59 static const unsigned CurrentVersion = 1;
     60 
     61 //----------------------------------------------------------------------------//
     62 // Global module index reader.
     63 //----------------------------------------------------------------------------//
     64 
     65 namespace {
     66 
     67 /// \brief Trait used to read the identifier index from the on-disk hash
     68 /// table.
     69 class IdentifierIndexReaderTrait {
     70 public:
     71   typedef StringRef external_key_type;
     72   typedef StringRef internal_key_type;
     73   typedef SmallVector<unsigned, 2> data_type;
     74 
     75   static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
     76     return a == b;
     77   }
     78 
     79   static unsigned ComputeHash(const internal_key_type& a) {
     80     return llvm::HashString(a);
     81   }
     82 
     83   static std::pair<unsigned, unsigned>
     84   ReadKeyDataLength(const unsigned char*& d) {
     85     using namespace clang::io;
     86     unsigned KeyLen = ReadUnalignedLE16(d);
     87     unsigned DataLen = ReadUnalignedLE16(d);
     88     return std::make_pair(KeyLen, DataLen);
     89   }
     90 
     91   static const internal_key_type&
     92   GetInternalKey(const external_key_type& x) { return x; }
     93 
     94   static const external_key_type&
     95   GetExternalKey(const internal_key_type& x) { return x; }
     96 
     97   static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
     98     return StringRef((const char *)d, n);
     99   }
    100 
    101   static data_type ReadData(const internal_key_type& k,
    102                             const unsigned char* d,
    103                             unsigned DataLen) {
    104     using namespace clang::io;
    105 
    106     data_type Result;
    107     while (DataLen > 0) {
    108       unsigned ID = ReadUnalignedLE32(d);
    109       Result.push_back(ID);
    110       DataLen -= 4;
    111     }
    112 
    113     return Result;
    114   }
    115 };
    116 
    117 typedef OnDiskChainedHashTable<IdentifierIndexReaderTrait> IdentifierIndexTable;
    118 
    119 }
    120 
    121 GlobalModuleIndex::GlobalModuleIndex(llvm::MemoryBuffer *Buffer,
    122                                      llvm::BitstreamCursor Cursor)
    123   : Buffer(Buffer), IdentifierIndex(),
    124     NumIdentifierLookups(), NumIdentifierLookupHits()
    125 {
    126   // Read the global index.
    127   bool InGlobalIndexBlock = false;
    128   bool Done = false;
    129   while (!Done) {
    130     llvm::BitstreamEntry Entry = Cursor.advance();
    131 
    132     switch (Entry.Kind) {
    133     case llvm::BitstreamEntry::Error:
    134       return;
    135 
    136     case llvm::BitstreamEntry::EndBlock:
    137       if (InGlobalIndexBlock) {
    138         InGlobalIndexBlock = false;
    139         Done = true;
    140         continue;
    141       }
    142       return;
    143 
    144 
    145     case llvm::BitstreamEntry::Record:
    146       // Entries in the global index block are handled below.
    147       if (InGlobalIndexBlock)
    148         break;
    149 
    150       return;
    151 
    152     case llvm::BitstreamEntry::SubBlock:
    153       if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
    154         if (Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
    155           return;
    156 
    157         InGlobalIndexBlock = true;
    158       } else if (Cursor.SkipBlock()) {
    159         return;
    160       }
    161       continue;
    162     }
    163 
    164     SmallVector<uint64_t, 64> Record;
    165     StringRef Blob;
    166     switch ((IndexRecordTypes)Cursor.readRecord(Entry.ID, Record, &Blob)) {
    167     case INDEX_METADATA:
    168       // Make sure that the version matches.
    169       if (Record.size() < 1 || Record[0] != CurrentVersion)
    170         return;
    171       break;
    172 
    173     case MODULE: {
    174       unsigned Idx = 0;
    175       unsigned ID = Record[Idx++];
    176 
    177       // Make room for this module's information.
    178       if (ID == Modules.size())
    179         Modules.push_back(ModuleInfo());
    180       else
    181         Modules.resize(ID + 1);
    182 
    183       // Size/modification time for this module file at the time the
    184       // global index was built.
    185       Modules[ID].Size = Record[Idx++];
    186       Modules[ID].ModTime = Record[Idx++];
    187 
    188       // File name.
    189       unsigned NameLen = Record[Idx++];
    190       Modules[ID].FileName.assign(Record.begin() + Idx,
    191                                   Record.begin() + Idx + NameLen);
    192       Idx += NameLen;
    193 
    194       // Dependencies
    195       unsigned NumDeps = Record[Idx++];
    196       Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(),
    197                                       Record.begin() + Idx,
    198                                       Record.begin() + Idx + NumDeps);
    199       Idx += NumDeps;
    200 
    201       // Make sure we're at the end of the record.
    202       assert(Idx == Record.size() && "More module info?");
    203 
    204       // Record this module as an unresolved module.
    205       UnresolvedModules[llvm::sys::path::stem(Modules[ID].FileName)] = ID;
    206       break;
    207     }
    208 
    209     case IDENTIFIER_INDEX:
    210       // Wire up the identifier index.
    211       if (Record[0]) {
    212         IdentifierIndex = IdentifierIndexTable::Create(
    213                             (const unsigned char *)Blob.data() + Record[0],
    214                             (const unsigned char *)Blob.data(),
    215                             IdentifierIndexReaderTrait());
    216       }
    217       break;
    218     }
    219   }
    220 }
    221 
    222 GlobalModuleIndex::~GlobalModuleIndex() { }
    223 
    224 std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode>
    225 GlobalModuleIndex::readIndex(StringRef Path) {
    226   // Load the index file, if it's there.
    227   llvm::SmallString<128> IndexPath;
    228   IndexPath += Path;
    229   llvm::sys::path::append(IndexPath, IndexFileName);
    230 
    231   llvm::OwningPtr<llvm::MemoryBuffer> Buffer;
    232   if (llvm::MemoryBuffer::getFile(IndexPath, Buffer) != llvm::errc::success)
    233     return std::make_pair((GlobalModuleIndex *)0, EC_NotFound);
    234 
    235   /// \brief The bitstream reader from which we'll read the AST file.
    236   llvm::BitstreamReader Reader((const unsigned char *)Buffer->getBufferStart(),
    237                                (const unsigned char *)Buffer->getBufferEnd());
    238 
    239   /// \brief The main bitstream cursor for the main block.
    240   llvm::BitstreamCursor Cursor(Reader);
    241 
    242   // Sniff for the signature.
    243   if (Cursor.Read(8) != 'B' ||
    244       Cursor.Read(8) != 'C' ||
    245       Cursor.Read(8) != 'G' ||
    246       Cursor.Read(8) != 'I') {
    247     return std::make_pair((GlobalModuleIndex *)0, EC_IOError);
    248   }
    249 
    250   return std::make_pair(new GlobalModuleIndex(Buffer.take(), Cursor), EC_None);
    251 }
    252 
    253 void
    254 GlobalModuleIndex::getKnownModules(SmallVectorImpl<ModuleFile *> &ModuleFiles) {
    255   ModuleFiles.clear();
    256   for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
    257     if (ModuleFile *MF = Modules[I].File)
    258       ModuleFiles.push_back(MF);
    259   }
    260 }
    261 
    262 void GlobalModuleIndex::getModuleDependencies(
    263        ModuleFile *File,
    264        SmallVectorImpl<ModuleFile *> &Dependencies) {
    265   // Look for information about this module file.
    266   llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
    267     = ModulesByFile.find(File);
    268   if (Known == ModulesByFile.end())
    269     return;
    270 
    271   // Record dependencies.
    272   Dependencies.clear();
    273   ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
    274   for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
    275     if (ModuleFile *MF = Modules[I].File)
    276       Dependencies.push_back(MF);
    277   }
    278 }
    279 
    280 bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
    281   Hits.clear();
    282 
    283   // If there's no identifier index, there is nothing we can do.
    284   if (!IdentifierIndex)
    285     return false;
    286 
    287   // Look into the identifier index.
    288   ++NumIdentifierLookups;
    289   IdentifierIndexTable &Table
    290     = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
    291   IdentifierIndexTable::iterator Known = Table.find(Name);
    292   if (Known == Table.end()) {
    293     return true;
    294   }
    295 
    296   SmallVector<unsigned, 2> ModuleIDs = *Known;
    297   for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
    298     if (ModuleFile *MF = Modules[ModuleIDs[I]].File)
    299       Hits.insert(MF);
    300   }
    301 
    302   ++NumIdentifierLookupHits;
    303   return true;
    304 }
    305 
    306 bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) {
    307   // Look for the module in the global module index based on the module name.
    308   StringRef Name = llvm::sys::path::stem(File->FileName);
    309   llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
    310   if (Known == UnresolvedModules.end()) {
    311     return true;
    312   }
    313 
    314   // Rectify this module with the global module index.
    315   ModuleInfo &Info = Modules[Known->second];
    316 
    317   //  If the size and modification time match what we expected, record this
    318   // module file.
    319   bool Failed = true;
    320   if (File->File->getSize() == Info.Size &&
    321       File->File->getModificationTime() == Info.ModTime) {
    322     Info.File = File;
    323     ModulesByFile[File] = Known->second;
    324 
    325     Failed = false;
    326   }
    327 
    328   // One way or another, we have resolved this module file.
    329   UnresolvedModules.erase(Known);
    330   return Failed;
    331 }
    332 
    333 void GlobalModuleIndex::printStats() {
    334   std::fprintf(stderr, "*** Global Module Index Statistics:\n");
    335   if (NumIdentifierLookups) {
    336     fprintf(stderr, "  %u / %u identifier lookups succeeded (%f%%)\n",
    337             NumIdentifierLookupHits, NumIdentifierLookups,
    338             (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
    339   }
    340   std::fprintf(stderr, "\n");
    341 }
    342 
    343 //----------------------------------------------------------------------------//
    344 // Global module index writer.
    345 //----------------------------------------------------------------------------//
    346 
    347 namespace {
    348   /// \brief Provides information about a specific module file.
    349   struct ModuleFileInfo {
    350     /// \brief The numberic ID for this module file.
    351     unsigned ID;
    352 
    353     /// \brief The set of modules on which this module depends. Each entry is
    354     /// a module ID.
    355     SmallVector<unsigned, 4> Dependencies;
    356   };
    357 
    358   /// \brief Builder that generates the global module index file.
    359   class GlobalModuleIndexBuilder {
    360     FileManager &FileMgr;
    361 
    362     /// \brief Mapping from files to module file information.
    363     typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
    364 
    365     /// \brief Information about each of the known module files.
    366     ModuleFilesMap ModuleFiles;
    367 
    368     /// \brief Mapping from identifiers to the list of module file IDs that
    369     /// consider this identifier to be interesting.
    370     typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
    371 
    372     /// \brief A mapping from all interesting identifiers to the set of module
    373     /// files in which those identifiers are considered interesting.
    374     InterestingIdentifierMap InterestingIdentifiers;
    375 
    376     /// \brief Write the block-info block for the global module index file.
    377     void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
    378 
    379     /// \brief Retrieve the module file information for the given file.
    380     ModuleFileInfo &getModuleFileInfo(const FileEntry *File) {
    381       llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
    382         = ModuleFiles.find(File);
    383       if (Known != ModuleFiles.end())
    384         return Known->second;
    385 
    386       unsigned NewID = ModuleFiles.size();
    387       ModuleFileInfo &Info = ModuleFiles[File];
    388       Info.ID = NewID;
    389       return Info;
    390     }
    391 
    392   public:
    393     explicit GlobalModuleIndexBuilder(FileManager &FileMgr) : FileMgr(FileMgr){}
    394 
    395     /// \brief Load the contents of the given module file into the builder.
    396     ///
    397     /// \returns true if an error occurred, false otherwise.
    398     bool loadModuleFile(const FileEntry *File);
    399 
    400     /// \brief Write the index to the given bitstream.
    401     void writeIndex(llvm::BitstreamWriter &Stream);
    402   };
    403 }
    404 
    405 static void emitBlockID(unsigned ID, const char *Name,
    406                         llvm::BitstreamWriter &Stream,
    407                         SmallVectorImpl<uint64_t> &Record) {
    408   Record.clear();
    409   Record.push_back(ID);
    410   Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
    411 
    412   // Emit the block name if present.
    413   if (Name == 0 || Name[0] == 0) return;
    414   Record.clear();
    415   while (*Name)
    416     Record.push_back(*Name++);
    417   Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
    418 }
    419 
    420 static void emitRecordID(unsigned ID, const char *Name,
    421                          llvm::BitstreamWriter &Stream,
    422                          SmallVectorImpl<uint64_t> &Record) {
    423   Record.clear();
    424   Record.push_back(ID);
    425   while (*Name)
    426     Record.push_back(*Name++);
    427   Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
    428 }
    429 
    430 void
    431 GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
    432   SmallVector<uint64_t, 64> Record;
    433   Stream.EnterSubblock(llvm::bitc::BLOCKINFO_BLOCK_ID, 3);
    434 
    435 #define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
    436 #define RECORD(X) emitRecordID(X, #X, Stream, Record)
    437   BLOCK(GLOBAL_INDEX_BLOCK);
    438   RECORD(INDEX_METADATA);
    439   RECORD(MODULE);
    440   RECORD(IDENTIFIER_INDEX);
    441 #undef RECORD
    442 #undef BLOCK
    443 
    444   Stream.ExitBlock();
    445 }
    446 
    447 namespace {
    448   class InterestingASTIdentifierLookupTrait
    449     : public serialization::reader::ASTIdentifierLookupTraitBase {
    450 
    451   public:
    452     /// \brief The identifier and whether it is "interesting".
    453     typedef std::pair<StringRef, bool> data_type;
    454 
    455     data_type ReadData(const internal_key_type& k,
    456                        const unsigned char* d,
    457                        unsigned DataLen) {
    458       // The first bit indicates whether this identifier is interesting.
    459       // That's all we care about.
    460       using namespace clang::io;
    461       unsigned RawID = ReadUnalignedLE32(d);
    462       bool IsInteresting = RawID & 0x01;
    463       return std::make_pair(k, IsInteresting);
    464     }
    465   };
    466 }
    467 
    468 bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
    469   // Open the module file.
    470   OwningPtr<llvm::MemoryBuffer> Buffer;
    471   std::string ErrorStr;
    472   Buffer.reset(FileMgr.getBufferForFile(File, &ErrorStr, /*isVolatile=*/true));
    473   if (!Buffer) {
    474     return true;
    475   }
    476 
    477   // Initialize the input stream
    478   llvm::BitstreamReader InStreamFile;
    479   llvm::BitstreamCursor InStream;
    480   InStreamFile.init((const unsigned char *)Buffer->getBufferStart(),
    481                   (const unsigned char *)Buffer->getBufferEnd());
    482   InStream.init(InStreamFile);
    483 
    484   // Sniff for the signature.
    485   if (InStream.Read(8) != 'C' ||
    486       InStream.Read(8) != 'P' ||
    487       InStream.Read(8) != 'C' ||
    488       InStream.Read(8) != 'H') {
    489     return true;
    490   }
    491 
    492   // Record this module file and assign it a unique ID (if it doesn't have
    493   // one already).
    494   unsigned ID = getModuleFileInfo(File).ID;
    495 
    496   // Search for the blocks and records we care about.
    497   enum { Other, ControlBlock, ASTBlock } State = Other;
    498   bool Done = false;
    499   while (!Done) {
    500     llvm::BitstreamEntry Entry = InStream.advance();
    501     switch (Entry.Kind) {
    502     case llvm::BitstreamEntry::Error:
    503       Done = true;
    504       continue;
    505 
    506     case llvm::BitstreamEntry::Record:
    507       // In the 'other' state, just skip the record. We don't care.
    508       if (State == Other) {
    509         InStream.skipRecord(Entry.ID);
    510         continue;
    511       }
    512 
    513       // Handle potentially-interesting records below.
    514       break;
    515 
    516     case llvm::BitstreamEntry::SubBlock:
    517       if (Entry.ID == CONTROL_BLOCK_ID) {
    518         if (InStream.EnterSubBlock(CONTROL_BLOCK_ID))
    519           return true;
    520 
    521         // Found the control block.
    522         State = ControlBlock;
    523         continue;
    524       }
    525 
    526       if (Entry.ID == AST_BLOCK_ID) {
    527         if (InStream.EnterSubBlock(AST_BLOCK_ID))
    528           return true;
    529 
    530         // Found the AST block.
    531         State = ASTBlock;
    532         continue;
    533       }
    534 
    535       if (InStream.SkipBlock())
    536         return true;
    537 
    538       continue;
    539 
    540     case llvm::BitstreamEntry::EndBlock:
    541       State = Other;
    542       continue;
    543     }
    544 
    545     // Read the given record.
    546     SmallVector<uint64_t, 64> Record;
    547     StringRef Blob;
    548     unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob);
    549 
    550     // Handle module dependencies.
    551     if (State == ControlBlock && Code == IMPORTS) {
    552       // Load each of the imported PCH files.
    553       unsigned Idx = 0, N = Record.size();
    554       while (Idx < N) {
    555         // Read information about the AST file.
    556 
    557         // Skip the imported kind
    558         ++Idx;
    559 
    560         // Skip the import location
    561         ++Idx;
    562 
    563         // Load stored size/modification time.
    564         off_t StoredSize = (off_t)Record[Idx++];
    565         time_t StoredModTime = (time_t)Record[Idx++];
    566 
    567         // Retrieve the imported file name.
    568         unsigned Length = Record[Idx++];
    569         SmallString<128> ImportedFile(Record.begin() + Idx,
    570                                       Record.begin() + Idx + Length);
    571         Idx += Length;
    572 
    573         // Find the imported module file.
    574         const FileEntry *DependsOnFile
    575           = FileMgr.getFile(ImportedFile, /*openFile=*/false,
    576                             /*cacheFailure=*/false);
    577         if (!DependsOnFile ||
    578             (StoredSize != DependsOnFile->getSize()) ||
    579             (StoredModTime != DependsOnFile->getModificationTime()))
    580           return true;
    581 
    582         // Record the dependency.
    583         unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
    584         getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
    585       }
    586 
    587       continue;
    588     }
    589 
    590     // Handle the identifier table
    591     if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) {
    592       typedef OnDiskChainedHashTable<InterestingASTIdentifierLookupTrait>
    593         InterestingIdentifierTable;
    594       llvm::OwningPtr<InterestingIdentifierTable>
    595         Table(InterestingIdentifierTable::Create(
    596                 (const unsigned char *)Blob.data() + Record[0],
    597                 (const unsigned char *)Blob.data()));
    598       for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
    599                                                      DEnd = Table->data_end();
    600            D != DEnd; ++D) {
    601         std::pair<StringRef, bool> Ident = *D;
    602         if (Ident.second)
    603           InterestingIdentifiers[Ident.first].push_back(ID);
    604         else
    605           (void)InterestingIdentifiers[Ident.first];
    606       }
    607     }
    608 
    609     // We don't care about this record.
    610   }
    611 
    612   return false;
    613 }
    614 
    615 namespace {
    616 
    617 /// \brief Trait used to generate the identifier index as an on-disk hash
    618 /// table.
    619 class IdentifierIndexWriterTrait {
    620 public:
    621   typedef StringRef key_type;
    622   typedef StringRef key_type_ref;
    623   typedef SmallVector<unsigned, 2> data_type;
    624   typedef const SmallVector<unsigned, 2> &data_type_ref;
    625 
    626   static unsigned ComputeHash(key_type_ref Key) {
    627     return llvm::HashString(Key);
    628   }
    629 
    630   std::pair<unsigned,unsigned>
    631   EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
    632     unsigned KeyLen = Key.size();
    633     unsigned DataLen = Data.size() * 4;
    634     clang::io::Emit16(Out, KeyLen);
    635     clang::io::Emit16(Out, DataLen);
    636     return std::make_pair(KeyLen, DataLen);
    637   }
    638 
    639   void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
    640     Out.write(Key.data(), KeyLen);
    641   }
    642 
    643   void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
    644                 unsigned DataLen) {
    645     for (unsigned I = 0, N = Data.size(); I != N; ++I)
    646       clang::io::Emit32(Out, Data[I]);
    647   }
    648 };
    649 
    650 }
    651 
    652 void GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
    653   using namespace llvm;
    654 
    655   // Emit the file header.
    656   Stream.Emit((unsigned)'B', 8);
    657   Stream.Emit((unsigned)'C', 8);
    658   Stream.Emit((unsigned)'G', 8);
    659   Stream.Emit((unsigned)'I', 8);
    660 
    661   // Write the block-info block, which describes the records in this bitcode
    662   // file.
    663   emitBlockInfoBlock(Stream);
    664 
    665   Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
    666 
    667   // Write the metadata.
    668   SmallVector<uint64_t, 2> Record;
    669   Record.push_back(CurrentVersion);
    670   Stream.EmitRecord(INDEX_METADATA, Record);
    671 
    672   // Write the set of known module files.
    673   for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
    674                                 MEnd = ModuleFiles.end();
    675        M != MEnd; ++M) {
    676     Record.clear();
    677     Record.push_back(M->second.ID);
    678     Record.push_back(M->first->getSize());
    679     Record.push_back(M->first->getModificationTime());
    680 
    681     // File name
    682     StringRef Name(M->first->getName());
    683     Record.push_back(Name.size());
    684     Record.append(Name.begin(), Name.end());
    685 
    686     // Dependencies
    687     Record.push_back(M->second.Dependencies.size());
    688     Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
    689     Stream.EmitRecord(MODULE, Record);
    690   }
    691 
    692   // Write the identifier -> module file mapping.
    693   {
    694     OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
    695     IdentifierIndexWriterTrait Trait;
    696 
    697     // Populate the hash table.
    698     for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
    699                                             IEnd = InterestingIdentifiers.end();
    700          I != IEnd; ++I) {
    701       Generator.insert(I->first(), I->second, Trait);
    702     }
    703 
    704     // Create the on-disk hash table in a buffer.
    705     SmallString<4096> IdentifierTable;
    706     uint32_t BucketOffset;
    707     {
    708       llvm::raw_svector_ostream Out(IdentifierTable);
    709       // Make sure that no bucket is at offset 0
    710       clang::io::Emit32(Out, 0);
    711       BucketOffset = Generator.Emit(Out, Trait);
    712     }
    713 
    714     // Create a blob abbreviation
    715     BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
    716     Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
    717     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
    718     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
    719     unsigned IDTableAbbrev = Stream.EmitAbbrev(Abbrev);
    720 
    721     // Write the identifier table
    722     Record.clear();
    723     Record.push_back(IDENTIFIER_INDEX);
    724     Record.push_back(BucketOffset);
    725     Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable.str());
    726   }
    727 
    728   Stream.ExitBlock();
    729 }
    730 
    731 GlobalModuleIndex::ErrorCode
    732 GlobalModuleIndex::writeIndex(FileManager &FileMgr, StringRef Path) {
    733   llvm::SmallString<128> IndexPath;
    734   IndexPath += Path;
    735   llvm::sys::path::append(IndexPath, IndexFileName);
    736 
    737   // Coordinate building the global index file with other processes that might
    738   // try to do the same.
    739   llvm::LockFileManager Locked(IndexPath);
    740   switch (Locked) {
    741   case llvm::LockFileManager::LFS_Error:
    742     return EC_IOError;
    743 
    744   case llvm::LockFileManager::LFS_Owned:
    745     // We're responsible for building the index ourselves. Do so below.
    746     break;
    747 
    748   case llvm::LockFileManager::LFS_Shared:
    749     // Someone else is responsible for building the index. We don't care
    750     // when they finish, so we're done.
    751     return EC_Building;
    752   }
    753 
    754   // The module index builder.
    755   GlobalModuleIndexBuilder Builder(FileMgr);
    756 
    757   // Load each of the module files.
    758   llvm::error_code EC;
    759   for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
    760        D != DEnd && !EC;
    761        D.increment(EC)) {
    762     // If this isn't a module file, we don't care.
    763     if (llvm::sys::path::extension(D->path()) != ".pcm") {
    764       // ... unless it's a .pcm.lock file, which indicates that someone is
    765       // in the process of rebuilding a module. They'll rebuild the index
    766       // at the end of that translation unit, so we don't have to.
    767       if (llvm::sys::path::extension(D->path()) == ".pcm.lock")
    768         return EC_Building;
    769 
    770       continue;
    771     }
    772 
    773     // If we can't find the module file, skip it.
    774     const FileEntry *ModuleFile = FileMgr.getFile(D->path());
    775     if (!ModuleFile)
    776       continue;
    777 
    778     // Load this module file.
    779     if (Builder.loadModuleFile(ModuleFile))
    780       return EC_IOError;
    781   }
    782 
    783   // The output buffer, into which the global index will be written.
    784   SmallVector<char, 16> OutputBuffer;
    785   {
    786     llvm::BitstreamWriter OutputStream(OutputBuffer);
    787     Builder.writeIndex(OutputStream);
    788   }
    789 
    790   // Write the global index file to a temporary file.
    791   llvm::SmallString<128> IndexTmpPath;
    792   int TmpFD;
    793   if (llvm::sys::fs::createUniqueFile(IndexPath + "-%%%%%%%%", TmpFD,
    794                                       IndexTmpPath))
    795     return EC_IOError;
    796 
    797   // Open the temporary global index file for output.
    798   llvm::raw_fd_ostream Out(TmpFD, true);
    799   if (Out.has_error())
    800     return EC_IOError;
    801 
    802   // Write the index.
    803   Out.write(OutputBuffer.data(), OutputBuffer.size());
    804   Out.close();
    805   if (Out.has_error())
    806     return EC_IOError;
    807 
    808   // Remove the old index file. It isn't relevant any more.
    809   bool OldIndexExisted;
    810   llvm::sys::fs::remove(IndexPath.str(), OldIndexExisted);
    811 
    812   // Rename the newly-written index file to the proper name.
    813   if (llvm::sys::fs::rename(IndexTmpPath.str(), IndexPath.str())) {
    814     // Rename failed; just remove the
    815     llvm::sys::fs::remove(IndexTmpPath.str(), OldIndexExisted);
    816     return EC_IOError;
    817   }
    818 
    819   // We're done.
    820   return EC_None;
    821 }
    822 
    823 namespace {
    824   class GlobalIndexIdentifierIterator : public IdentifierIterator {
    825     /// \brief The current position within the identifier lookup table.
    826     IdentifierIndexTable::key_iterator Current;
    827 
    828     /// \brief The end position within the identifier lookup table.
    829     IdentifierIndexTable::key_iterator End;
    830 
    831   public:
    832     explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
    833       Current = Idx.key_begin();
    834       End = Idx.key_end();
    835     }
    836 
    837     virtual StringRef Next() {
    838       if (Current == End)
    839         return StringRef();
    840 
    841       StringRef Result = *Current;
    842       ++Current;
    843       return Result;
    844     }
    845   };
    846 }
    847 
    848 IdentifierIterator *GlobalModuleIndex::createIdentifierIterator() const {
    849   IdentifierIndexTable &Table =
    850     *static_cast<IdentifierIndexTable *>(IdentifierIndex);
    851   return new GlobalIndexIdentifierIterator(Table);
    852 }
    853