1 //===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file implements the GlobalModuleIndex class. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "ASTReaderInternals.h" 15 #include "clang/Basic/FileManager.h" 16 #include "clang/Lex/HeaderSearch.h" 17 #include "clang/Serialization/ASTBitCodes.h" 18 #include "clang/Serialization/GlobalModuleIndex.h" 19 #include "clang/Serialization/Module.h" 20 #include "llvm/ADT/DenseMap.h" 21 #include "llvm/ADT/MapVector.h" 22 #include "llvm/ADT/SmallString.h" 23 #include "llvm/ADT/StringExtras.h" 24 #include "llvm/Bitcode/BitstreamReader.h" 25 #include "llvm/Bitcode/BitstreamWriter.h" 26 #include "llvm/Support/FileSystem.h" 27 #include "llvm/Support/LockFileManager.h" 28 #include "llvm/Support/MemoryBuffer.h" 29 #include "llvm/Support/OnDiskHashTable.h" 30 #include "llvm/Support/Path.h" 31 #include <cstdio> 32 using namespace clang; 33 using namespace serialization; 34 35 //----------------------------------------------------------------------------// 36 // Shared constants 37 //----------------------------------------------------------------------------// 38 namespace { 39 enum { 40 /// \brief The block containing the index. 41 GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID 42 }; 43 44 /// \brief Describes the record types in the index. 45 enum IndexRecordTypes { 46 /// \brief Contains version information and potentially other metadata, 47 /// used to determine if we can read this global index file. 48 INDEX_METADATA, 49 /// \brief Describes a module, including its file name and dependencies. 50 MODULE, 51 /// \brief The index for identifiers. 52 IDENTIFIER_INDEX 53 }; 54 } 55 56 /// \brief The name of the global index file. 57 static const char * const IndexFileName = "modules.idx"; 58 59 /// \brief The global index file version. 60 static const unsigned CurrentVersion = 1; 61 62 //----------------------------------------------------------------------------// 63 // Global module index reader. 64 //----------------------------------------------------------------------------// 65 66 namespace { 67 68 /// \brief Trait used to read the identifier index from the on-disk hash 69 /// table. 70 class IdentifierIndexReaderTrait { 71 public: 72 typedef StringRef external_key_type; 73 typedef StringRef internal_key_type; 74 typedef SmallVector<unsigned, 2> data_type; 75 typedef unsigned hash_value_type; 76 typedef unsigned offset_type; 77 78 static bool EqualKey(const internal_key_type& a, const internal_key_type& b) { 79 return a == b; 80 } 81 82 static hash_value_type ComputeHash(const internal_key_type& a) { 83 return llvm::HashString(a); 84 } 85 86 static std::pair<unsigned, unsigned> 87 ReadKeyDataLength(const unsigned char*& d) { 88 using namespace llvm::support; 89 unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d); 90 unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d); 91 return std::make_pair(KeyLen, DataLen); 92 } 93 94 static const internal_key_type& 95 GetInternalKey(const external_key_type& x) { return x; } 96 97 static const external_key_type& 98 GetExternalKey(const internal_key_type& x) { return x; } 99 100 static internal_key_type ReadKey(const unsigned char* d, unsigned n) { 101 return StringRef((const char *)d, n); 102 } 103 104 static data_type ReadData(const internal_key_type& k, 105 const unsigned char* d, 106 unsigned DataLen) { 107 using namespace llvm::support; 108 109 data_type Result; 110 while (DataLen > 0) { 111 unsigned ID = endian::readNext<uint32_t, little, unaligned>(d); 112 Result.push_back(ID); 113 DataLen -= 4; 114 } 115 116 return Result; 117 } 118 }; 119 120 typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait> 121 IdentifierIndexTable; 122 123 } 124 125 GlobalModuleIndex::GlobalModuleIndex(std::unique_ptr<llvm::MemoryBuffer> Buffer, 126 llvm::BitstreamCursor Cursor) 127 : Buffer(std::move(Buffer)), IdentifierIndex(), NumIdentifierLookups(), 128 NumIdentifierLookupHits() { 129 // Read the global index. 130 bool InGlobalIndexBlock = false; 131 bool Done = false; 132 while (!Done) { 133 llvm::BitstreamEntry Entry = Cursor.advance(); 134 135 switch (Entry.Kind) { 136 case llvm::BitstreamEntry::Error: 137 return; 138 139 case llvm::BitstreamEntry::EndBlock: 140 if (InGlobalIndexBlock) { 141 InGlobalIndexBlock = false; 142 Done = true; 143 continue; 144 } 145 return; 146 147 148 case llvm::BitstreamEntry::Record: 149 // Entries in the global index block are handled below. 150 if (InGlobalIndexBlock) 151 break; 152 153 return; 154 155 case llvm::BitstreamEntry::SubBlock: 156 if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) { 157 if (Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID)) 158 return; 159 160 InGlobalIndexBlock = true; 161 } else if (Cursor.SkipBlock()) { 162 return; 163 } 164 continue; 165 } 166 167 SmallVector<uint64_t, 64> Record; 168 StringRef Blob; 169 switch ((IndexRecordTypes)Cursor.readRecord(Entry.ID, Record, &Blob)) { 170 case INDEX_METADATA: 171 // Make sure that the version matches. 172 if (Record.size() < 1 || Record[0] != CurrentVersion) 173 return; 174 break; 175 176 case MODULE: { 177 unsigned Idx = 0; 178 unsigned ID = Record[Idx++]; 179 180 // Make room for this module's information. 181 if (ID == Modules.size()) 182 Modules.push_back(ModuleInfo()); 183 else 184 Modules.resize(ID + 1); 185 186 // Size/modification time for this module file at the time the 187 // global index was built. 188 Modules[ID].Size = Record[Idx++]; 189 Modules[ID].ModTime = Record[Idx++]; 190 191 // File name. 192 unsigned NameLen = Record[Idx++]; 193 Modules[ID].FileName.assign(Record.begin() + Idx, 194 Record.begin() + Idx + NameLen); 195 Idx += NameLen; 196 197 // Dependencies 198 unsigned NumDeps = Record[Idx++]; 199 Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(), 200 Record.begin() + Idx, 201 Record.begin() + Idx + NumDeps); 202 Idx += NumDeps; 203 204 // Make sure we're at the end of the record. 205 assert(Idx == Record.size() && "More module info?"); 206 207 // Record this module as an unresolved module. 208 // FIXME: this doesn't work correctly for module names containing path 209 // separators. 210 StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName); 211 // Remove the -<hash of ModuleMapPath> 212 ModuleName = ModuleName.rsplit('-').first; 213 UnresolvedModules[ModuleName] = ID; 214 break; 215 } 216 217 case IDENTIFIER_INDEX: 218 // Wire up the identifier index. 219 if (Record[0]) { 220 IdentifierIndex = IdentifierIndexTable::Create( 221 (const unsigned char *)Blob.data() + Record[0], 222 (const unsigned char *)Blob.data() + sizeof(uint32_t), 223 (const unsigned char *)Blob.data(), IdentifierIndexReaderTrait()); 224 } 225 break; 226 } 227 } 228 } 229 230 GlobalModuleIndex::~GlobalModuleIndex() { 231 delete static_cast<IdentifierIndexTable *>(IdentifierIndex); 232 } 233 234 std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode> 235 GlobalModuleIndex::readIndex(StringRef Path) { 236 // Load the index file, if it's there. 237 llvm::SmallString<128> IndexPath; 238 IndexPath += Path; 239 llvm::sys::path::append(IndexPath, IndexFileName); 240 241 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr = 242 llvm::MemoryBuffer::getFile(IndexPath.c_str()); 243 if (!BufferOrErr) 244 return std::make_pair(nullptr, EC_NotFound); 245 std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get()); 246 247 /// \brief The bitstream reader from which we'll read the AST file. 248 llvm::BitstreamReader Reader((const unsigned char *)Buffer->getBufferStart(), 249 (const unsigned char *)Buffer->getBufferEnd()); 250 251 /// \brief The main bitstream cursor for the main block. 252 llvm::BitstreamCursor Cursor(Reader); 253 254 // Sniff for the signature. 255 if (Cursor.Read(8) != 'B' || 256 Cursor.Read(8) != 'C' || 257 Cursor.Read(8) != 'G' || 258 Cursor.Read(8) != 'I') { 259 return std::make_pair(nullptr, EC_IOError); 260 } 261 262 return std::make_pair(new GlobalModuleIndex(std::move(Buffer), Cursor), 263 EC_None); 264 } 265 266 void 267 GlobalModuleIndex::getKnownModules(SmallVectorImpl<ModuleFile *> &ModuleFiles) { 268 ModuleFiles.clear(); 269 for (unsigned I = 0, N = Modules.size(); I != N; ++I) { 270 if (ModuleFile *MF = Modules[I].File) 271 ModuleFiles.push_back(MF); 272 } 273 } 274 275 void GlobalModuleIndex::getModuleDependencies( 276 ModuleFile *File, 277 SmallVectorImpl<ModuleFile *> &Dependencies) { 278 // Look for information about this module file. 279 llvm::DenseMap<ModuleFile *, unsigned>::iterator Known 280 = ModulesByFile.find(File); 281 if (Known == ModulesByFile.end()) 282 return; 283 284 // Record dependencies. 285 Dependencies.clear(); 286 ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies; 287 for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) { 288 if (ModuleFile *MF = Modules[I].File) 289 Dependencies.push_back(MF); 290 } 291 } 292 293 bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) { 294 Hits.clear(); 295 296 // If there's no identifier index, there is nothing we can do. 297 if (!IdentifierIndex) 298 return false; 299 300 // Look into the identifier index. 301 ++NumIdentifierLookups; 302 IdentifierIndexTable &Table 303 = *static_cast<IdentifierIndexTable *>(IdentifierIndex); 304 IdentifierIndexTable::iterator Known = Table.find(Name); 305 if (Known == Table.end()) { 306 return true; 307 } 308 309 SmallVector<unsigned, 2> ModuleIDs = *Known; 310 for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) { 311 if (ModuleFile *MF = Modules[ModuleIDs[I]].File) 312 Hits.insert(MF); 313 } 314 315 ++NumIdentifierLookupHits; 316 return true; 317 } 318 319 bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) { 320 // Look for the module in the global module index based on the module name. 321 StringRef Name = File->ModuleName; 322 llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name); 323 if (Known == UnresolvedModules.end()) { 324 return true; 325 } 326 327 // Rectify this module with the global module index. 328 ModuleInfo &Info = Modules[Known->second]; 329 330 // If the size and modification time match what we expected, record this 331 // module file. 332 bool Failed = true; 333 if (File->File->getSize() == Info.Size && 334 File->File->getModificationTime() == Info.ModTime) { 335 Info.File = File; 336 ModulesByFile[File] = Known->second; 337 338 Failed = false; 339 } 340 341 // One way or another, we have resolved this module file. 342 UnresolvedModules.erase(Known); 343 return Failed; 344 } 345 346 void GlobalModuleIndex::printStats() { 347 std::fprintf(stderr, "*** Global Module Index Statistics:\n"); 348 if (NumIdentifierLookups) { 349 fprintf(stderr, " %u / %u identifier lookups succeeded (%f%%)\n", 350 NumIdentifierLookupHits, NumIdentifierLookups, 351 (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups); 352 } 353 std::fprintf(stderr, "\n"); 354 } 355 356 void GlobalModuleIndex::dump() { 357 llvm::errs() << "*** Global Module Index Dump:\n"; 358 llvm::errs() << "Module files:\n"; 359 for (auto &MI : Modules) { 360 llvm::errs() << "** " << MI.FileName << "\n"; 361 if (MI.File) 362 MI.File->dump(); 363 else 364 llvm::errs() << "\n"; 365 } 366 llvm::errs() << "\n"; 367 } 368 369 //----------------------------------------------------------------------------// 370 // Global module index writer. 371 //----------------------------------------------------------------------------// 372 373 namespace { 374 /// \brief Provides information about a specific module file. 375 struct ModuleFileInfo { 376 /// \brief The numberic ID for this module file. 377 unsigned ID; 378 379 /// \brief The set of modules on which this module depends. Each entry is 380 /// a module ID. 381 SmallVector<unsigned, 4> Dependencies; 382 }; 383 384 /// \brief Builder that generates the global module index file. 385 class GlobalModuleIndexBuilder { 386 FileManager &FileMgr; 387 388 /// \brief Mapping from files to module file information. 389 typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap; 390 391 /// \brief Information about each of the known module files. 392 ModuleFilesMap ModuleFiles; 393 394 /// \brief Mapping from identifiers to the list of module file IDs that 395 /// consider this identifier to be interesting. 396 typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap; 397 398 /// \brief A mapping from all interesting identifiers to the set of module 399 /// files in which those identifiers are considered interesting. 400 InterestingIdentifierMap InterestingIdentifiers; 401 402 /// \brief Write the block-info block for the global module index file. 403 void emitBlockInfoBlock(llvm::BitstreamWriter &Stream); 404 405 /// \brief Retrieve the module file information for the given file. 406 ModuleFileInfo &getModuleFileInfo(const FileEntry *File) { 407 llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known 408 = ModuleFiles.find(File); 409 if (Known != ModuleFiles.end()) 410 return Known->second; 411 412 unsigned NewID = ModuleFiles.size(); 413 ModuleFileInfo &Info = ModuleFiles[File]; 414 Info.ID = NewID; 415 return Info; 416 } 417 418 public: 419 explicit GlobalModuleIndexBuilder(FileManager &FileMgr) : FileMgr(FileMgr){} 420 421 /// \brief Load the contents of the given module file into the builder. 422 /// 423 /// \returns true if an error occurred, false otherwise. 424 bool loadModuleFile(const FileEntry *File); 425 426 /// \brief Write the index to the given bitstream. 427 void writeIndex(llvm::BitstreamWriter &Stream); 428 }; 429 } 430 431 static void emitBlockID(unsigned ID, const char *Name, 432 llvm::BitstreamWriter &Stream, 433 SmallVectorImpl<uint64_t> &Record) { 434 Record.clear(); 435 Record.push_back(ID); 436 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record); 437 438 // Emit the block name if present. 439 if (!Name || Name[0] == 0) return; 440 Record.clear(); 441 while (*Name) 442 Record.push_back(*Name++); 443 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record); 444 } 445 446 static void emitRecordID(unsigned ID, const char *Name, 447 llvm::BitstreamWriter &Stream, 448 SmallVectorImpl<uint64_t> &Record) { 449 Record.clear(); 450 Record.push_back(ID); 451 while (*Name) 452 Record.push_back(*Name++); 453 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record); 454 } 455 456 void 457 GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) { 458 SmallVector<uint64_t, 64> Record; 459 Stream.EnterSubblock(llvm::bitc::BLOCKINFO_BLOCK_ID, 3); 460 461 #define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record) 462 #define RECORD(X) emitRecordID(X, #X, Stream, Record) 463 BLOCK(GLOBAL_INDEX_BLOCK); 464 RECORD(INDEX_METADATA); 465 RECORD(MODULE); 466 RECORD(IDENTIFIER_INDEX); 467 #undef RECORD 468 #undef BLOCK 469 470 Stream.ExitBlock(); 471 } 472 473 namespace { 474 class InterestingASTIdentifierLookupTrait 475 : public serialization::reader::ASTIdentifierLookupTraitBase { 476 477 public: 478 /// \brief The identifier and whether it is "interesting". 479 typedef std::pair<StringRef, bool> data_type; 480 481 data_type ReadData(const internal_key_type& k, 482 const unsigned char* d, 483 unsigned DataLen) { 484 // The first bit indicates whether this identifier is interesting. 485 // That's all we care about. 486 using namespace llvm::support; 487 unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d); 488 bool IsInteresting = RawID & 0x01; 489 return std::make_pair(k, IsInteresting); 490 } 491 }; 492 } 493 494 bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) { 495 // Open the module file. 496 497 auto Buffer = FileMgr.getBufferForFile(File, /*isVolatile=*/true); 498 if (!Buffer) { 499 return true; 500 } 501 502 // Initialize the input stream 503 llvm::BitstreamReader InStreamFile; 504 InStreamFile.init((const unsigned char *)(*Buffer)->getBufferStart(), 505 (const unsigned char *)(*Buffer)->getBufferEnd()); 506 llvm::BitstreamCursor InStream(InStreamFile); 507 508 // Sniff for the signature. 509 if (InStream.Read(8) != 'C' || 510 InStream.Read(8) != 'P' || 511 InStream.Read(8) != 'C' || 512 InStream.Read(8) != 'H') { 513 return true; 514 } 515 516 // Record this module file and assign it a unique ID (if it doesn't have 517 // one already). 518 unsigned ID = getModuleFileInfo(File).ID; 519 520 // Search for the blocks and records we care about. 521 enum { Other, ControlBlock, ASTBlock } State = Other; 522 bool Done = false; 523 while (!Done) { 524 llvm::BitstreamEntry Entry = InStream.advance(); 525 switch (Entry.Kind) { 526 case llvm::BitstreamEntry::Error: 527 Done = true; 528 continue; 529 530 case llvm::BitstreamEntry::Record: 531 // In the 'other' state, just skip the record. We don't care. 532 if (State == Other) { 533 InStream.skipRecord(Entry.ID); 534 continue; 535 } 536 537 // Handle potentially-interesting records below. 538 break; 539 540 case llvm::BitstreamEntry::SubBlock: 541 if (Entry.ID == CONTROL_BLOCK_ID) { 542 if (InStream.EnterSubBlock(CONTROL_BLOCK_ID)) 543 return true; 544 545 // Found the control block. 546 State = ControlBlock; 547 continue; 548 } 549 550 if (Entry.ID == AST_BLOCK_ID) { 551 if (InStream.EnterSubBlock(AST_BLOCK_ID)) 552 return true; 553 554 // Found the AST block. 555 State = ASTBlock; 556 continue; 557 } 558 559 if (InStream.SkipBlock()) 560 return true; 561 562 continue; 563 564 case llvm::BitstreamEntry::EndBlock: 565 State = Other; 566 continue; 567 } 568 569 // Read the given record. 570 SmallVector<uint64_t, 64> Record; 571 StringRef Blob; 572 unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob); 573 574 // Handle module dependencies. 575 if (State == ControlBlock && Code == IMPORTS) { 576 // Load each of the imported PCH files. 577 unsigned Idx = 0, N = Record.size(); 578 while (Idx < N) { 579 // Read information about the AST file. 580 581 // Skip the imported kind 582 ++Idx; 583 584 // Skip the import location 585 ++Idx; 586 587 // Load stored size/modification time. 588 off_t StoredSize = (off_t)Record[Idx++]; 589 time_t StoredModTime = (time_t)Record[Idx++]; 590 591 // Skip the stored signature. 592 // FIXME: we could read the signature out of the import and validate it. 593 Idx++; 594 595 // Retrieve the imported file name. 596 unsigned Length = Record[Idx++]; 597 SmallString<128> ImportedFile(Record.begin() + Idx, 598 Record.begin() + Idx + Length); 599 Idx += Length; 600 601 // Find the imported module file. 602 const FileEntry *DependsOnFile 603 = FileMgr.getFile(ImportedFile, /*openFile=*/false, 604 /*cacheFailure=*/false); 605 if (!DependsOnFile || 606 (StoredSize != DependsOnFile->getSize()) || 607 (StoredModTime != DependsOnFile->getModificationTime())) 608 return true; 609 610 // Record the dependency. 611 unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID; 612 getModuleFileInfo(File).Dependencies.push_back(DependsOnID); 613 } 614 615 continue; 616 } 617 618 // Handle the identifier table 619 if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) { 620 typedef llvm::OnDiskIterableChainedHashTable< 621 InterestingASTIdentifierLookupTrait> InterestingIdentifierTable; 622 std::unique_ptr<InterestingIdentifierTable> Table( 623 InterestingIdentifierTable::Create( 624 (const unsigned char *)Blob.data() + Record[0], 625 (const unsigned char *)Blob.data() + sizeof(uint32_t), 626 (const unsigned char *)Blob.data())); 627 for (InterestingIdentifierTable::data_iterator D = Table->data_begin(), 628 DEnd = Table->data_end(); 629 D != DEnd; ++D) { 630 std::pair<StringRef, bool> Ident = *D; 631 if (Ident.second) 632 InterestingIdentifiers[Ident.first].push_back(ID); 633 else 634 (void)InterestingIdentifiers[Ident.first]; 635 } 636 } 637 638 // We don't care about this record. 639 } 640 641 return false; 642 } 643 644 namespace { 645 646 /// \brief Trait used to generate the identifier index as an on-disk hash 647 /// table. 648 class IdentifierIndexWriterTrait { 649 public: 650 typedef StringRef key_type; 651 typedef StringRef key_type_ref; 652 typedef SmallVector<unsigned, 2> data_type; 653 typedef const SmallVector<unsigned, 2> &data_type_ref; 654 typedef unsigned hash_value_type; 655 typedef unsigned offset_type; 656 657 static hash_value_type ComputeHash(key_type_ref Key) { 658 return llvm::HashString(Key); 659 } 660 661 std::pair<unsigned,unsigned> 662 EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) { 663 using namespace llvm::support; 664 endian::Writer<little> LE(Out); 665 unsigned KeyLen = Key.size(); 666 unsigned DataLen = Data.size() * 4; 667 LE.write<uint16_t>(KeyLen); 668 LE.write<uint16_t>(DataLen); 669 return std::make_pair(KeyLen, DataLen); 670 } 671 672 void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) { 673 Out.write(Key.data(), KeyLen); 674 } 675 676 void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data, 677 unsigned DataLen) { 678 using namespace llvm::support; 679 for (unsigned I = 0, N = Data.size(); I != N; ++I) 680 endian::Writer<little>(Out).write<uint32_t>(Data[I]); 681 } 682 }; 683 684 } 685 686 void GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) { 687 using namespace llvm; 688 689 // Emit the file header. 690 Stream.Emit((unsigned)'B', 8); 691 Stream.Emit((unsigned)'C', 8); 692 Stream.Emit((unsigned)'G', 8); 693 Stream.Emit((unsigned)'I', 8); 694 695 // Write the block-info block, which describes the records in this bitcode 696 // file. 697 emitBlockInfoBlock(Stream); 698 699 Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3); 700 701 // Write the metadata. 702 SmallVector<uint64_t, 2> Record; 703 Record.push_back(CurrentVersion); 704 Stream.EmitRecord(INDEX_METADATA, Record); 705 706 // Write the set of known module files. 707 for (ModuleFilesMap::iterator M = ModuleFiles.begin(), 708 MEnd = ModuleFiles.end(); 709 M != MEnd; ++M) { 710 Record.clear(); 711 Record.push_back(M->second.ID); 712 Record.push_back(M->first->getSize()); 713 Record.push_back(M->first->getModificationTime()); 714 715 // File name 716 StringRef Name(M->first->getName()); 717 Record.push_back(Name.size()); 718 Record.append(Name.begin(), Name.end()); 719 720 // Dependencies 721 Record.push_back(M->second.Dependencies.size()); 722 Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end()); 723 Stream.EmitRecord(MODULE, Record); 724 } 725 726 // Write the identifier -> module file mapping. 727 { 728 llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator; 729 IdentifierIndexWriterTrait Trait; 730 731 // Populate the hash table. 732 for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(), 733 IEnd = InterestingIdentifiers.end(); 734 I != IEnd; ++I) { 735 Generator.insert(I->first(), I->second, Trait); 736 } 737 738 // Create the on-disk hash table in a buffer. 739 SmallString<4096> IdentifierTable; 740 uint32_t BucketOffset; 741 { 742 using namespace llvm::support; 743 llvm::raw_svector_ostream Out(IdentifierTable); 744 // Make sure that no bucket is at offset 0 745 endian::Writer<little>(Out).write<uint32_t>(0); 746 BucketOffset = Generator.Emit(Out, Trait); 747 } 748 749 // Create a blob abbreviation 750 BitCodeAbbrev *Abbrev = new BitCodeAbbrev(); 751 Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX)); 752 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); 753 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); 754 unsigned IDTableAbbrev = Stream.EmitAbbrev(Abbrev); 755 756 // Write the identifier table 757 Record.clear(); 758 Record.push_back(IDENTIFIER_INDEX); 759 Record.push_back(BucketOffset); 760 Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable); 761 } 762 763 Stream.ExitBlock(); 764 } 765 766 GlobalModuleIndex::ErrorCode 767 GlobalModuleIndex::writeIndex(FileManager &FileMgr, StringRef Path) { 768 llvm::SmallString<128> IndexPath; 769 IndexPath += Path; 770 llvm::sys::path::append(IndexPath, IndexFileName); 771 772 // Coordinate building the global index file with other processes that might 773 // try to do the same. 774 llvm::LockFileManager Locked(IndexPath); 775 switch (Locked) { 776 case llvm::LockFileManager::LFS_Error: 777 return EC_IOError; 778 779 case llvm::LockFileManager::LFS_Owned: 780 // We're responsible for building the index ourselves. Do so below. 781 break; 782 783 case llvm::LockFileManager::LFS_Shared: 784 // Someone else is responsible for building the index. We don't care 785 // when they finish, so we're done. 786 return EC_Building; 787 } 788 789 // The module index builder. 790 GlobalModuleIndexBuilder Builder(FileMgr); 791 792 // Load each of the module files. 793 std::error_code EC; 794 for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd; 795 D != DEnd && !EC; 796 D.increment(EC)) { 797 // If this isn't a module file, we don't care. 798 if (llvm::sys::path::extension(D->path()) != ".pcm") { 799 // ... unless it's a .pcm.lock file, which indicates that someone is 800 // in the process of rebuilding a module. They'll rebuild the index 801 // at the end of that translation unit, so we don't have to. 802 if (llvm::sys::path::extension(D->path()) == ".pcm.lock") 803 return EC_Building; 804 805 continue; 806 } 807 808 // If we can't find the module file, skip it. 809 const FileEntry *ModuleFile = FileMgr.getFile(D->path()); 810 if (!ModuleFile) 811 continue; 812 813 // Load this module file. 814 if (Builder.loadModuleFile(ModuleFile)) 815 return EC_IOError; 816 } 817 818 // The output buffer, into which the global index will be written. 819 SmallVector<char, 16> OutputBuffer; 820 { 821 llvm::BitstreamWriter OutputStream(OutputBuffer); 822 Builder.writeIndex(OutputStream); 823 } 824 825 // Write the global index file to a temporary file. 826 llvm::SmallString<128> IndexTmpPath; 827 int TmpFD; 828 if (llvm::sys::fs::createUniqueFile(IndexPath + "-%%%%%%%%", TmpFD, 829 IndexTmpPath)) 830 return EC_IOError; 831 832 // Open the temporary global index file for output. 833 llvm::raw_fd_ostream Out(TmpFD, true); 834 if (Out.has_error()) 835 return EC_IOError; 836 837 // Write the index. 838 Out.write(OutputBuffer.data(), OutputBuffer.size()); 839 Out.close(); 840 if (Out.has_error()) 841 return EC_IOError; 842 843 // Remove the old index file. It isn't relevant any more. 844 llvm::sys::fs::remove(IndexPath); 845 846 // Rename the newly-written index file to the proper name. 847 if (llvm::sys::fs::rename(IndexTmpPath, IndexPath)) { 848 // Rename failed; just remove the 849 llvm::sys::fs::remove(IndexTmpPath); 850 return EC_IOError; 851 } 852 853 // We're done. 854 return EC_None; 855 } 856 857 namespace { 858 class GlobalIndexIdentifierIterator : public IdentifierIterator { 859 /// \brief The current position within the identifier lookup table. 860 IdentifierIndexTable::key_iterator Current; 861 862 /// \brief The end position within the identifier lookup table. 863 IdentifierIndexTable::key_iterator End; 864 865 public: 866 explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) { 867 Current = Idx.key_begin(); 868 End = Idx.key_end(); 869 } 870 871 StringRef Next() override { 872 if (Current == End) 873 return StringRef(); 874 875 StringRef Result = *Current; 876 ++Current; 877 return Result; 878 } 879 }; 880 } 881 882 IdentifierIterator *GlobalModuleIndex::createIdentifierIterator() const { 883 IdentifierIndexTable &Table = 884 *static_cast<IdentifierIndexTable *>(IdentifierIndex); 885 return new GlobalIndexIdentifierIterator(Table); 886 } 887