1 //===--- FileManager.cpp - File System Probing and Caching ----------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file implements the FileManager interface. 11 // 12 //===----------------------------------------------------------------------===// 13 // 14 // TODO: This should index all interesting directories with dirent calls. 15 // getdirentries ? 16 // opendir/readdir_r/closedir ? 17 // 18 //===----------------------------------------------------------------------===// 19 20 #include "clang/Basic/FileManager.h" 21 #include "clang/Basic/FileSystemStatCache.h" 22 #include "llvm/ADT/SmallString.h" 23 #include "llvm/ADT/StringExtras.h" 24 #include "llvm/Support/FileSystem.h" 25 #include "llvm/Support/MemoryBuffer.h" 26 #include "llvm/Support/raw_ostream.h" 27 #include "llvm/Support/Path.h" 28 #include "llvm/Support/system_error.h" 29 #include "llvm/Config/config.h" 30 #include <map> 31 #include <set> 32 #include <string> 33 34 // FIXME: This is terrible, we need this for ::close. 35 #if !defined(_MSC_VER) && !defined(__MINGW32__) 36 #include <unistd.h> 37 #include <sys/uio.h> 38 #else 39 #include <io.h> 40 #endif 41 using namespace clang; 42 43 // FIXME: Enhance libsystem to support inode and other fields. 44 #include <sys/stat.h> 45 46 /// NON_EXISTENT_DIR - A special value distinct from null that is used to 47 /// represent a dir name that doesn't exist on the disk. 48 #define NON_EXISTENT_DIR reinterpret_cast<DirectoryEntry*>((intptr_t)-1) 49 50 /// NON_EXISTENT_FILE - A special value distinct from null that is used to 51 /// represent a filename that doesn't exist on the disk. 52 #define NON_EXISTENT_FILE reinterpret_cast<FileEntry*>((intptr_t)-1) 53 54 55 FileEntry::~FileEntry() { 56 // If this FileEntry owns an open file descriptor that never got used, close 57 // it. 58 if (FD != -1) ::close(FD); 59 } 60 61 //===----------------------------------------------------------------------===// 62 // Windows. 63 //===----------------------------------------------------------------------===// 64 65 #ifdef LLVM_ON_WIN32 66 67 namespace { 68 static std::string GetFullPath(const char *relPath) { 69 char *absPathStrPtr = _fullpath(NULL, relPath, 0); 70 assert(absPathStrPtr && "_fullpath() returned NULL!"); 71 72 std::string absPath(absPathStrPtr); 73 74 free(absPathStrPtr); 75 return absPath; 76 } 77 } 78 79 class FileManager::UniqueDirContainer { 80 /// UniqueDirs - Cache from full path to existing directories/files. 81 /// 82 llvm::StringMap<DirectoryEntry> UniqueDirs; 83 84 public: 85 /// getDirectory - Return an existing DirectoryEntry with the given 86 /// name if there is already one; otherwise create and return a 87 /// default-constructed DirectoryEntry. 88 DirectoryEntry &getDirectory(const char *Name, 89 const struct stat & /*StatBuf*/) { 90 std::string FullPath(GetFullPath(Name)); 91 return UniqueDirs.GetOrCreateValue(FullPath).getValue(); 92 } 93 94 size_t size() const { return UniqueDirs.size(); } 95 }; 96 97 class FileManager::UniqueFileContainer { 98 /// UniqueFiles - Cache from full path to existing directories/files. 99 /// 100 llvm::StringMap<FileEntry, llvm::BumpPtrAllocator> UniqueFiles; 101 102 public: 103 /// getFile - Return an existing FileEntry with the given name if 104 /// there is already one; otherwise create and return a 105 /// default-constructed FileEntry. 106 FileEntry &getFile(const char *Name, const struct stat & /*StatBuf*/) { 107 std::string FullPath(GetFullPath(Name)); 108 109 // LowercaseString because Windows filesystem is case insensitive. 110 FullPath = llvm::LowercaseString(FullPath); 111 return UniqueFiles.GetOrCreateValue(FullPath).getValue(); 112 } 113 114 size_t size() const { return UniqueFiles.size(); } 115 }; 116 117 //===----------------------------------------------------------------------===// 118 // Unix-like Systems. 119 //===----------------------------------------------------------------------===// 120 121 #else 122 123 class FileManager::UniqueDirContainer { 124 /// UniqueDirs - Cache from ID's to existing directories/files. 125 std::map<std::pair<dev_t, ino_t>, DirectoryEntry> UniqueDirs; 126 127 public: 128 /// getDirectory - Return an existing DirectoryEntry with the given 129 /// ID's if there is already one; otherwise create and return a 130 /// default-constructed DirectoryEntry. 131 DirectoryEntry &getDirectory(const char * /*Name*/, 132 const struct stat &StatBuf) { 133 return UniqueDirs[std::make_pair(StatBuf.st_dev, StatBuf.st_ino)]; 134 } 135 136 size_t size() const { return UniqueDirs.size(); } 137 }; 138 139 class FileManager::UniqueFileContainer { 140 /// UniqueFiles - Cache from ID's to existing directories/files. 141 std::set<FileEntry> UniqueFiles; 142 143 public: 144 /// getFile - Return an existing FileEntry with the given ID's if 145 /// there is already one; otherwise create and return a 146 /// default-constructed FileEntry. 147 FileEntry &getFile(const char * /*Name*/, const struct stat &StatBuf) { 148 return 149 const_cast<FileEntry&>( 150 *UniqueFiles.insert(FileEntry(StatBuf.st_dev, 151 StatBuf.st_ino, 152 StatBuf.st_mode)).first); 153 } 154 155 size_t size() const { return UniqueFiles.size(); } 156 }; 157 158 #endif 159 160 //===----------------------------------------------------------------------===// 161 // Common logic. 162 //===----------------------------------------------------------------------===// 163 164 FileManager::FileManager(const FileSystemOptions &FSO) 165 : FileSystemOpts(FSO), 166 UniqueRealDirs(*new UniqueDirContainer()), 167 UniqueRealFiles(*new UniqueFileContainer()), 168 SeenDirEntries(64), SeenFileEntries(64), NextFileUID(0) { 169 NumDirLookups = NumFileLookups = 0; 170 NumDirCacheMisses = NumFileCacheMisses = 0; 171 } 172 173 FileManager::~FileManager() { 174 delete &UniqueRealDirs; 175 delete &UniqueRealFiles; 176 for (unsigned i = 0, e = VirtualFileEntries.size(); i != e; ++i) 177 delete VirtualFileEntries[i]; 178 for (unsigned i = 0, e = VirtualDirectoryEntries.size(); i != e; ++i) 179 delete VirtualDirectoryEntries[i]; 180 } 181 182 void FileManager::addStatCache(FileSystemStatCache *statCache, 183 bool AtBeginning) { 184 assert(statCache && "No stat cache provided?"); 185 if (AtBeginning || StatCache.get() == 0) { 186 statCache->setNextStatCache(StatCache.take()); 187 StatCache.reset(statCache); 188 return; 189 } 190 191 FileSystemStatCache *LastCache = StatCache.get(); 192 while (LastCache->getNextStatCache()) 193 LastCache = LastCache->getNextStatCache(); 194 195 LastCache->setNextStatCache(statCache); 196 } 197 198 void FileManager::removeStatCache(FileSystemStatCache *statCache) { 199 if (!statCache) 200 return; 201 202 if (StatCache.get() == statCache) { 203 // This is the first stat cache. 204 StatCache.reset(StatCache->takeNextStatCache()); 205 return; 206 } 207 208 // Find the stat cache in the list. 209 FileSystemStatCache *PrevCache = StatCache.get(); 210 while (PrevCache && PrevCache->getNextStatCache() != statCache) 211 PrevCache = PrevCache->getNextStatCache(); 212 213 assert(PrevCache && "Stat cache not found for removal"); 214 PrevCache->setNextStatCache(statCache->getNextStatCache()); 215 } 216 217 /// \brief Retrieve the directory that the given file name resides in. 218 /// Filename can point to either a real file or a virtual file. 219 static const DirectoryEntry *getDirectoryFromFile(FileManager &FileMgr, 220 StringRef Filename, 221 bool CacheFailure) { 222 if (Filename.empty()) 223 return NULL; 224 225 if (llvm::sys::path::is_separator(Filename[Filename.size() - 1])) 226 return NULL; // If Filename is a directory. 227 228 StringRef DirName = llvm::sys::path::parent_path(Filename); 229 // Use the current directory if file has no path component. 230 if (DirName.empty()) 231 DirName = "."; 232 233 return FileMgr.getDirectory(DirName, CacheFailure); 234 } 235 236 /// Add all ancestors of the given path (pointing to either a file or 237 /// a directory) as virtual directories. 238 void FileManager::addAncestorsAsVirtualDirs(StringRef Path) { 239 StringRef DirName = llvm::sys::path::parent_path(Path); 240 if (DirName.empty()) 241 return; 242 243 llvm::StringMapEntry<DirectoryEntry *> &NamedDirEnt = 244 SeenDirEntries.GetOrCreateValue(DirName); 245 246 // When caching a virtual directory, we always cache its ancestors 247 // at the same time. Therefore, if DirName is already in the cache, 248 // we don't need to recurse as its ancestors must also already be in 249 // the cache. 250 if (NamedDirEnt.getValue()) 251 return; 252 253 // Add the virtual directory to the cache. 254 DirectoryEntry *UDE = new DirectoryEntry; 255 UDE->Name = NamedDirEnt.getKeyData(); 256 NamedDirEnt.setValue(UDE); 257 VirtualDirectoryEntries.push_back(UDE); 258 259 // Recursively add the other ancestors. 260 addAncestorsAsVirtualDirs(DirName); 261 } 262 263 /// getDirectory - Lookup, cache, and verify the specified directory 264 /// (real or virtual). This returns NULL if the directory doesn't 265 /// exist. 266 /// 267 const DirectoryEntry *FileManager::getDirectory(StringRef DirName, 268 bool CacheFailure) { 269 ++NumDirLookups; 270 llvm::StringMapEntry<DirectoryEntry *> &NamedDirEnt = 271 SeenDirEntries.GetOrCreateValue(DirName); 272 273 // See if there was already an entry in the map. Note that the map 274 // contains both virtual and real directories. 275 if (NamedDirEnt.getValue()) 276 return NamedDirEnt.getValue() == NON_EXISTENT_DIR 277 ? 0 : NamedDirEnt.getValue(); 278 279 ++NumDirCacheMisses; 280 281 // By default, initialize it to invalid. 282 NamedDirEnt.setValue(NON_EXISTENT_DIR); 283 284 // Get the null-terminated directory name as stored as the key of the 285 // SeenDirEntries map. 286 const char *InterndDirName = NamedDirEnt.getKeyData(); 287 288 // Check to see if the directory exists. 289 struct stat StatBuf; 290 if (getStatValue(InterndDirName, StatBuf, 0/*directory lookup*/)) { 291 // There's no real directory at the given path. 292 if (!CacheFailure) 293 SeenDirEntries.erase(DirName); 294 return 0; 295 } 296 297 // It exists. See if we have already opened a directory with the 298 // same inode (this occurs on Unix-like systems when one dir is 299 // symlinked to another, for example) or the same path (on 300 // Windows). 301 DirectoryEntry &UDE = UniqueRealDirs.getDirectory(InterndDirName, StatBuf); 302 303 NamedDirEnt.setValue(&UDE); 304 if (!UDE.getName()) { 305 // We don't have this directory yet, add it. We use the string 306 // key from the SeenDirEntries map as the string. 307 UDE.Name = InterndDirName; 308 } 309 310 return &UDE; 311 } 312 313 /// getFile - Lookup, cache, and verify the specified file (real or 314 /// virtual). This returns NULL if the file doesn't exist. 315 /// 316 const FileEntry *FileManager::getFile(StringRef Filename, bool openFile, 317 bool CacheFailure) { 318 ++NumFileLookups; 319 320 // See if there is already an entry in the map. 321 llvm::StringMapEntry<FileEntry *> &NamedFileEnt = 322 SeenFileEntries.GetOrCreateValue(Filename); 323 324 // See if there is already an entry in the map. 325 if (NamedFileEnt.getValue()) 326 return NamedFileEnt.getValue() == NON_EXISTENT_FILE 327 ? 0 : NamedFileEnt.getValue(); 328 329 ++NumFileCacheMisses; 330 331 // By default, initialize it to invalid. 332 NamedFileEnt.setValue(NON_EXISTENT_FILE); 333 334 // Get the null-terminated file name as stored as the key of the 335 // SeenFileEntries map. 336 const char *InterndFileName = NamedFileEnt.getKeyData(); 337 338 // Look up the directory for the file. When looking up something like 339 // sys/foo.h we'll discover all of the search directories that have a 'sys' 340 // subdirectory. This will let us avoid having to waste time on known-to-fail 341 // searches when we go to find sys/bar.h, because all the search directories 342 // without a 'sys' subdir will get a cached failure result. 343 const DirectoryEntry *DirInfo = getDirectoryFromFile(*this, Filename, 344 CacheFailure); 345 if (DirInfo == 0) { // Directory doesn't exist, file can't exist. 346 if (!CacheFailure) 347 SeenFileEntries.erase(Filename); 348 349 return 0; 350 } 351 352 // FIXME: Use the directory info to prune this, before doing the stat syscall. 353 // FIXME: This will reduce the # syscalls. 354 355 // Nope, there isn't. Check to see if the file exists. 356 int FileDescriptor = -1; 357 struct stat StatBuf; 358 if (getStatValue(InterndFileName, StatBuf, &FileDescriptor)) { 359 // There's no real file at the given path. 360 if (!CacheFailure) 361 SeenFileEntries.erase(Filename); 362 363 return 0; 364 } 365 366 if (FileDescriptor != -1 && !openFile) { 367 close(FileDescriptor); 368 FileDescriptor = -1; 369 } 370 371 // It exists. See if we have already opened a file with the same inode. 372 // This occurs when one dir is symlinked to another, for example. 373 FileEntry &UFE = UniqueRealFiles.getFile(InterndFileName, StatBuf); 374 375 NamedFileEnt.setValue(&UFE); 376 if (UFE.getName()) { // Already have an entry with this inode, return it. 377 // If the stat process opened the file, close it to avoid a FD leak. 378 if (FileDescriptor != -1) 379 close(FileDescriptor); 380 381 return &UFE; 382 } 383 384 // Otherwise, we don't have this directory yet, add it. 385 // FIXME: Change the name to be a char* that points back to the 386 // 'SeenFileEntries' key. 387 UFE.Name = InterndFileName; 388 UFE.Size = StatBuf.st_size; 389 UFE.ModTime = StatBuf.st_mtime; 390 UFE.Dir = DirInfo; 391 UFE.UID = NextFileUID++; 392 UFE.FD = FileDescriptor; 393 return &UFE; 394 } 395 396 const FileEntry * 397 FileManager::getVirtualFile(StringRef Filename, off_t Size, 398 time_t ModificationTime) { 399 ++NumFileLookups; 400 401 // See if there is already an entry in the map. 402 llvm::StringMapEntry<FileEntry *> &NamedFileEnt = 403 SeenFileEntries.GetOrCreateValue(Filename); 404 405 // See if there is already an entry in the map. 406 if (NamedFileEnt.getValue() && NamedFileEnt.getValue() != NON_EXISTENT_FILE) 407 return NamedFileEnt.getValue(); 408 409 ++NumFileCacheMisses; 410 411 // By default, initialize it to invalid. 412 NamedFileEnt.setValue(NON_EXISTENT_FILE); 413 414 addAncestorsAsVirtualDirs(Filename); 415 FileEntry *UFE = 0; 416 417 // Now that all ancestors of Filename are in the cache, the 418 // following call is guaranteed to find the DirectoryEntry from the 419 // cache. 420 const DirectoryEntry *DirInfo = getDirectoryFromFile(*this, Filename, 421 /*CacheFailure=*/true); 422 assert(DirInfo && 423 "The directory of a virtual file should already be in the cache."); 424 425 // Check to see if the file exists. If so, drop the virtual file 426 int FileDescriptor = -1; 427 struct stat StatBuf; 428 const char *InterndFileName = NamedFileEnt.getKeyData(); 429 if (getStatValue(InterndFileName, StatBuf, &FileDescriptor) == 0) { 430 // If the stat process opened the file, close it to avoid a FD leak. 431 if (FileDescriptor != -1) 432 close(FileDescriptor); 433 434 StatBuf.st_size = Size; 435 StatBuf.st_mtime = ModificationTime; 436 UFE = &UniqueRealFiles.getFile(InterndFileName, StatBuf); 437 438 NamedFileEnt.setValue(UFE); 439 440 // If we had already opened this file, close it now so we don't 441 // leak the descriptor. We're not going to use the file 442 // descriptor anyway, since this is a virtual file. 443 if (UFE->FD != -1) { 444 close(UFE->FD); 445 UFE->FD = -1; 446 } 447 448 // If we already have an entry with this inode, return it. 449 if (UFE->getName()) 450 return UFE; 451 } 452 453 if (!UFE) { 454 UFE = new FileEntry(); 455 VirtualFileEntries.push_back(UFE); 456 NamedFileEnt.setValue(UFE); 457 } 458 459 UFE->Name = InterndFileName; 460 UFE->Size = Size; 461 UFE->ModTime = ModificationTime; 462 UFE->Dir = DirInfo; 463 UFE->UID = NextFileUID++; 464 UFE->FD = -1; 465 return UFE; 466 } 467 468 void FileManager::FixupRelativePath(SmallVectorImpl<char> &path) const { 469 StringRef pathRef(path.data(), path.size()); 470 471 if (FileSystemOpts.WorkingDir.empty() 472 || llvm::sys::path::is_absolute(pathRef)) 473 return; 474 475 llvm::SmallString<128> NewPath(FileSystemOpts.WorkingDir); 476 llvm::sys::path::append(NewPath, pathRef); 477 path = NewPath; 478 } 479 480 llvm::MemoryBuffer *FileManager:: 481 getBufferForFile(const FileEntry *Entry, std::string *ErrorStr) { 482 llvm::OwningPtr<llvm::MemoryBuffer> Result; 483 llvm::error_code ec; 484 485 const char *Filename = Entry->getName(); 486 // If the file is already open, use the open file descriptor. 487 if (Entry->FD != -1) { 488 ec = llvm::MemoryBuffer::getOpenFile(Entry->FD, Filename, Result, 489 Entry->getSize()); 490 if (ErrorStr) 491 *ErrorStr = ec.message(); 492 493 close(Entry->FD); 494 Entry->FD = -1; 495 return Result.take(); 496 } 497 498 // Otherwise, open the file. 499 500 if (FileSystemOpts.WorkingDir.empty()) { 501 ec = llvm::MemoryBuffer::getFile(Filename, Result, Entry->getSize()); 502 if (ec && ErrorStr) 503 *ErrorStr = ec.message(); 504 return Result.take(); 505 } 506 507 llvm::SmallString<128> FilePath(Entry->getName()); 508 FixupRelativePath(FilePath); 509 ec = llvm::MemoryBuffer::getFile(FilePath.str(), Result, Entry->getSize()); 510 if (ec && ErrorStr) 511 *ErrorStr = ec.message(); 512 return Result.take(); 513 } 514 515 llvm::MemoryBuffer *FileManager:: 516 getBufferForFile(StringRef Filename, std::string *ErrorStr) { 517 llvm::OwningPtr<llvm::MemoryBuffer> Result; 518 llvm::error_code ec; 519 if (FileSystemOpts.WorkingDir.empty()) { 520 ec = llvm::MemoryBuffer::getFile(Filename, Result); 521 if (ec && ErrorStr) 522 *ErrorStr = ec.message(); 523 return Result.take(); 524 } 525 526 llvm::SmallString<128> FilePath(Filename); 527 FixupRelativePath(FilePath); 528 ec = llvm::MemoryBuffer::getFile(FilePath.c_str(), Result); 529 if (ec && ErrorStr) 530 *ErrorStr = ec.message(); 531 return Result.take(); 532 } 533 534 /// getStatValue - Get the 'stat' information for the specified path, 535 /// using the cache to accelerate it if possible. This returns true 536 /// if the path points to a virtual file or does not exist, or returns 537 /// false if it's an existent real file. If FileDescriptor is NULL, 538 /// do directory look-up instead of file look-up. 539 bool FileManager::getStatValue(const char *Path, struct stat &StatBuf, 540 int *FileDescriptor) { 541 // FIXME: FileSystemOpts shouldn't be passed in here, all paths should be 542 // absolute! 543 if (FileSystemOpts.WorkingDir.empty()) 544 return FileSystemStatCache::get(Path, StatBuf, FileDescriptor, 545 StatCache.get()); 546 547 llvm::SmallString<128> FilePath(Path); 548 FixupRelativePath(FilePath); 549 550 return FileSystemStatCache::get(FilePath.c_str(), StatBuf, FileDescriptor, 551 StatCache.get()); 552 } 553 554 bool FileManager::getNoncachedStatValue(StringRef Path, 555 struct stat &StatBuf) { 556 llvm::SmallString<128> FilePath(Path); 557 FixupRelativePath(FilePath); 558 559 return ::stat(FilePath.c_str(), &StatBuf) != 0; 560 } 561 562 void FileManager::GetUniqueIDMapping( 563 SmallVectorImpl<const FileEntry *> &UIDToFiles) const { 564 UIDToFiles.clear(); 565 UIDToFiles.resize(NextFileUID); 566 567 // Map file entries 568 for (llvm::StringMap<FileEntry*, llvm::BumpPtrAllocator>::const_iterator 569 FE = SeenFileEntries.begin(), FEEnd = SeenFileEntries.end(); 570 FE != FEEnd; ++FE) 571 if (FE->getValue() && FE->getValue() != NON_EXISTENT_FILE) 572 UIDToFiles[FE->getValue()->getUID()] = FE->getValue(); 573 574 // Map virtual file entries 575 for (SmallVector<FileEntry*, 4>::const_iterator 576 VFE = VirtualFileEntries.begin(), VFEEnd = VirtualFileEntries.end(); 577 VFE != VFEEnd; ++VFE) 578 if (*VFE && *VFE != NON_EXISTENT_FILE) 579 UIDToFiles[(*VFE)->getUID()] = *VFE; 580 } 581 582 583 void FileManager::PrintStats() const { 584 llvm::errs() << "\n*** File Manager Stats:\n"; 585 llvm::errs() << UniqueRealFiles.size() << " real files found, " 586 << UniqueRealDirs.size() << " real dirs found.\n"; 587 llvm::errs() << VirtualFileEntries.size() << " virtual files found, " 588 << VirtualDirectoryEntries.size() << " virtual dirs found.\n"; 589 llvm::errs() << NumDirLookups << " dir lookups, " 590 << NumDirCacheMisses << " dir cache misses.\n"; 591 llvm::errs() << NumFileLookups << " file lookups, " 592 << NumFileCacheMisses << " file cache misses.\n"; 593 594 //llvm::errs() << PagesMapped << BytesOfPagesMapped << FSLookups; 595 } 596