1 //===- VirtualFileSystem.cpp - Virtual File System Layer --------*- C++ -*-===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // This file implements the VirtualFileSystem interface. 10 //===----------------------------------------------------------------------===// 11 12 #include "clang/Basic/VirtualFileSystem.h" 13 #include "llvm/ADT/DenseMap.h" 14 #include "llvm/ADT/iterator_range.h" 15 #include "llvm/ADT/STLExtras.h" 16 #include "llvm/ADT/StringExtras.h" 17 #include "llvm/ADT/StringSet.h" 18 #include "llvm/Support/Errc.h" 19 #include "llvm/Support/MemoryBuffer.h" 20 #include "llvm/Support/Path.h" 21 #include "llvm/Support/YAMLParser.h" 22 #include <atomic> 23 #include <memory> 24 25 using namespace clang; 26 using namespace clang::vfs; 27 using namespace llvm; 28 using llvm::sys::fs::file_status; 29 using llvm::sys::fs::file_type; 30 using llvm::sys::fs::perms; 31 using llvm::sys::fs::UniqueID; 32 33 Status::Status(const file_status &Status) 34 : UID(Status.getUniqueID()), MTime(Status.getLastModificationTime()), 35 User(Status.getUser()), Group(Status.getGroup()), Size(Status.getSize()), 36 Type(Status.type()), Perms(Status.permissions()), IsVFSMapped(false) {} 37 38 Status::Status(StringRef Name, StringRef ExternalName, UniqueID UID, 39 sys::TimeValue MTime, uint32_t User, uint32_t Group, 40 uint64_t Size, file_type Type, perms Perms) 41 : Name(Name), UID(UID), MTime(MTime), User(User), Group(Group), Size(Size), 42 Type(Type), Perms(Perms), IsVFSMapped(false) {} 43 44 bool Status::equivalent(const Status &Other) const { 45 return getUniqueID() == Other.getUniqueID(); 46 } 47 bool Status::isDirectory() const { 48 return Type == file_type::directory_file; 49 } 50 bool Status::isRegularFile() const { 51 return Type == file_type::regular_file; 52 } 53 bool Status::isOther() const { 54 return exists() && !isRegularFile() && !isDirectory() && !isSymlink(); 55 } 56 bool Status::isSymlink() const { 57 return Type == file_type::symlink_file; 58 } 59 bool Status::isStatusKnown() const { 60 return Type != file_type::status_error; 61 } 62 bool Status::exists() const { 63 return isStatusKnown() && Type != file_type::file_not_found; 64 } 65 66 File::~File() {} 67 68 FileSystem::~FileSystem() {} 69 70 std::error_code FileSystem::getBufferForFile( 71 const llvm::Twine &Name, std::unique_ptr<MemoryBuffer> &Result, 72 int64_t FileSize, bool RequiresNullTerminator, bool IsVolatile) { 73 std::unique_ptr<File> F; 74 if (std::error_code EC = openFileForRead(Name, F)) 75 return EC; 76 77 std::error_code EC = 78 F->getBuffer(Name, Result, FileSize, RequiresNullTerminator, IsVolatile); 79 return EC; 80 } 81 82 //===-----------------------------------------------------------------------===/ 83 // RealFileSystem implementation 84 //===-----------------------------------------------------------------------===/ 85 86 namespace { 87 /// \brief Wrapper around a raw file descriptor. 88 class RealFile : public File { 89 int FD; 90 Status S; 91 friend class RealFileSystem; 92 RealFile(int FD) : FD(FD) { 93 assert(FD >= 0 && "Invalid or inactive file descriptor"); 94 } 95 96 public: 97 ~RealFile(); 98 ErrorOr<Status> status() override; 99 std::error_code getBuffer(const Twine &Name, 100 std::unique_ptr<MemoryBuffer> &Result, 101 int64_t FileSize = -1, 102 bool RequiresNullTerminator = true, 103 bool IsVolatile = false) override; 104 std::error_code close() override; 105 void setName(StringRef Name) override; 106 }; 107 } // end anonymous namespace 108 RealFile::~RealFile() { close(); } 109 110 ErrorOr<Status> RealFile::status() { 111 assert(FD != -1 && "cannot stat closed file"); 112 if (!S.isStatusKnown()) { 113 file_status RealStatus; 114 if (std::error_code EC = sys::fs::status(FD, RealStatus)) 115 return EC; 116 Status NewS(RealStatus); 117 NewS.setName(S.getName()); 118 S = std::move(NewS); 119 } 120 return S; 121 } 122 123 std::error_code RealFile::getBuffer(const Twine &Name, 124 std::unique_ptr<MemoryBuffer> &Result, 125 int64_t FileSize, 126 bool RequiresNullTerminator, 127 bool IsVolatile) { 128 assert(FD != -1 && "cannot get buffer for closed file"); 129 ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr = 130 MemoryBuffer::getOpenFile(FD, Name.str().c_str(), FileSize, 131 RequiresNullTerminator, IsVolatile); 132 if (std::error_code EC = BufferOrErr.getError()) 133 return EC; 134 Result = std::move(BufferOrErr.get()); 135 return std::error_code(); 136 } 137 138 // FIXME: This is terrible, we need this for ::close. 139 #if !defined(_MSC_VER) && !defined(__MINGW32__) 140 #include <unistd.h> 141 #include <sys/uio.h> 142 #else 143 #include <io.h> 144 #ifndef S_ISFIFO 145 #define S_ISFIFO(x) (0) 146 #endif 147 #endif 148 std::error_code RealFile::close() { 149 if (::close(FD)) 150 return std::error_code(errno, std::generic_category()); 151 FD = -1; 152 return std::error_code(); 153 } 154 155 void RealFile::setName(StringRef Name) { 156 S.setName(Name); 157 } 158 159 namespace { 160 /// \brief The file system according to your operating system. 161 class RealFileSystem : public FileSystem { 162 public: 163 ErrorOr<Status> status(const Twine &Path) override; 164 std::error_code openFileForRead(const Twine &Path, 165 std::unique_ptr<File> &Result) override; 166 directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override; 167 }; 168 } // end anonymous namespace 169 170 ErrorOr<Status> RealFileSystem::status(const Twine &Path) { 171 sys::fs::file_status RealStatus; 172 if (std::error_code EC = sys::fs::status(Path, RealStatus)) 173 return EC; 174 Status Result(RealStatus); 175 Result.setName(Path.str()); 176 return Result; 177 } 178 179 std::error_code RealFileSystem::openFileForRead(const Twine &Name, 180 std::unique_ptr<File> &Result) { 181 int FD; 182 if (std::error_code EC = sys::fs::openFileForRead(Name, FD)) 183 return EC; 184 Result.reset(new RealFile(FD)); 185 Result->setName(Name.str()); 186 return std::error_code(); 187 } 188 189 IntrusiveRefCntPtr<FileSystem> vfs::getRealFileSystem() { 190 static IntrusiveRefCntPtr<FileSystem> FS = new RealFileSystem(); 191 return FS; 192 } 193 194 namespace { 195 class RealFSDirIter : public clang::vfs::detail::DirIterImpl { 196 std::string Path; 197 llvm::sys::fs::directory_iterator Iter; 198 public: 199 RealFSDirIter(const Twine &_Path, std::error_code &EC) 200 : Path(_Path.str()), Iter(Path, EC) { 201 if (!EC && Iter != llvm::sys::fs::directory_iterator()) { 202 llvm::sys::fs::file_status S; 203 EC = Iter->status(S); 204 if (!EC) { 205 CurrentEntry = Status(S); 206 CurrentEntry.setName(Iter->path()); 207 } 208 } 209 } 210 211 std::error_code increment() override { 212 std::error_code EC; 213 Iter.increment(EC); 214 if (EC) { 215 return EC; 216 } else if (Iter == llvm::sys::fs::directory_iterator()) { 217 CurrentEntry = Status(); 218 } else { 219 llvm::sys::fs::file_status S; 220 EC = Iter->status(S); 221 CurrentEntry = Status(S); 222 CurrentEntry.setName(Iter->path()); 223 } 224 return EC; 225 } 226 }; 227 } 228 229 directory_iterator RealFileSystem::dir_begin(const Twine &Dir, 230 std::error_code &EC) { 231 return directory_iterator(std::make_shared<RealFSDirIter>(Dir, EC)); 232 } 233 234 //===-----------------------------------------------------------------------===/ 235 // OverlayFileSystem implementation 236 //===-----------------------------------------------------------------------===/ 237 OverlayFileSystem::OverlayFileSystem(IntrusiveRefCntPtr<FileSystem> BaseFS) { 238 pushOverlay(BaseFS); 239 } 240 241 void OverlayFileSystem::pushOverlay(IntrusiveRefCntPtr<FileSystem> FS) { 242 FSList.push_back(FS); 243 } 244 245 ErrorOr<Status> OverlayFileSystem::status(const Twine &Path) { 246 // FIXME: handle symlinks that cross file systems 247 for (iterator I = overlays_begin(), E = overlays_end(); I != E; ++I) { 248 ErrorOr<Status> Status = (*I)->status(Path); 249 if (Status || Status.getError() != llvm::errc::no_such_file_or_directory) 250 return Status; 251 } 252 return make_error_code(llvm::errc::no_such_file_or_directory); 253 } 254 255 std::error_code 256 OverlayFileSystem::openFileForRead(const llvm::Twine &Path, 257 std::unique_ptr<File> &Result) { 258 // FIXME: handle symlinks that cross file systems 259 for (iterator I = overlays_begin(), E = overlays_end(); I != E; ++I) { 260 std::error_code EC = (*I)->openFileForRead(Path, Result); 261 if (!EC || EC != llvm::errc::no_such_file_or_directory) 262 return EC; 263 } 264 return make_error_code(llvm::errc::no_such_file_or_directory); 265 } 266 267 clang::vfs::detail::DirIterImpl::~DirIterImpl() { } 268 269 namespace { 270 class OverlayFSDirIterImpl : public clang::vfs::detail::DirIterImpl { 271 OverlayFileSystem &Overlays; 272 std::string Path; 273 OverlayFileSystem::iterator CurrentFS; 274 directory_iterator CurrentDirIter; 275 llvm::StringSet<> SeenNames; 276 277 std::error_code incrementFS() { 278 assert(CurrentFS != Overlays.overlays_end() && "incrementing past end"); 279 ++CurrentFS; 280 for (auto E = Overlays.overlays_end(); CurrentFS != E; ++CurrentFS) { 281 std::error_code EC; 282 CurrentDirIter = (*CurrentFS)->dir_begin(Path, EC); 283 if (EC && EC != errc::no_such_file_or_directory) 284 return EC; 285 if (CurrentDirIter != directory_iterator()) 286 break; // found 287 } 288 return std::error_code(); 289 } 290 291 std::error_code incrementDirIter(bool IsFirstTime) { 292 assert((IsFirstTime || CurrentDirIter != directory_iterator()) && 293 "incrementing past end"); 294 std::error_code EC; 295 if (!IsFirstTime) 296 CurrentDirIter.increment(EC); 297 if (!EC && CurrentDirIter == directory_iterator()) 298 EC = incrementFS(); 299 return EC; 300 } 301 302 std::error_code incrementImpl(bool IsFirstTime) { 303 while (true) { 304 std::error_code EC = incrementDirIter(IsFirstTime); 305 if (EC || CurrentDirIter == directory_iterator()) { 306 CurrentEntry = Status(); 307 return EC; 308 } 309 CurrentEntry = *CurrentDirIter; 310 StringRef Name = llvm::sys::path::filename(CurrentEntry.getName()); 311 if (SeenNames.insert(Name)) 312 return EC; // name not seen before 313 } 314 llvm_unreachable("returned above"); 315 } 316 317 public: 318 OverlayFSDirIterImpl(const Twine &Path, OverlayFileSystem &FS, 319 std::error_code &EC) 320 : Overlays(FS), Path(Path.str()), CurrentFS(Overlays.overlays_begin()) { 321 CurrentDirIter = (*CurrentFS)->dir_begin(Path, EC); 322 EC = incrementImpl(true); 323 } 324 325 std::error_code increment() override { return incrementImpl(false); } 326 }; 327 } // end anonymous namespace 328 329 directory_iterator OverlayFileSystem::dir_begin(const Twine &Dir, 330 std::error_code &EC) { 331 return directory_iterator( 332 std::make_shared<OverlayFSDirIterImpl>(Dir, *this, EC)); 333 } 334 335 //===-----------------------------------------------------------------------===/ 336 // VFSFromYAML implementation 337 //===-----------------------------------------------------------------------===/ 338 339 // Allow DenseMap<StringRef, ...>. This is useful below because we know all the 340 // strings are literals and will outlive the map, and there is no reason to 341 // store them. 342 namespace llvm { 343 template<> 344 struct DenseMapInfo<StringRef> { 345 // This assumes that "" will never be a valid key. 346 static inline StringRef getEmptyKey() { return StringRef(""); } 347 static inline StringRef getTombstoneKey() { return StringRef(); } 348 static unsigned getHashValue(StringRef Val) { return HashString(Val); } 349 static bool isEqual(StringRef LHS, StringRef RHS) { return LHS == RHS; } 350 }; 351 } 352 353 namespace { 354 355 enum EntryKind { 356 EK_Directory, 357 EK_File 358 }; 359 360 /// \brief A single file or directory in the VFS. 361 class Entry { 362 EntryKind Kind; 363 std::string Name; 364 365 public: 366 virtual ~Entry(); 367 Entry(EntryKind K, StringRef Name) : Kind(K), Name(Name) {} 368 StringRef getName() const { return Name; } 369 EntryKind getKind() const { return Kind; } 370 }; 371 372 class DirectoryEntry : public Entry { 373 std::vector<Entry *> Contents; 374 Status S; 375 376 public: 377 virtual ~DirectoryEntry(); 378 DirectoryEntry(StringRef Name, std::vector<Entry *> Contents, Status S) 379 : Entry(EK_Directory, Name), Contents(std::move(Contents)), 380 S(std::move(S)) {} 381 Status getStatus() { return S; } 382 typedef std::vector<Entry *>::iterator iterator; 383 iterator contents_begin() { return Contents.begin(); } 384 iterator contents_end() { return Contents.end(); } 385 static bool classof(const Entry *E) { return E->getKind() == EK_Directory; } 386 }; 387 388 class FileEntry : public Entry { 389 public: 390 enum NameKind { 391 NK_NotSet, 392 NK_External, 393 NK_Virtual 394 }; 395 private: 396 std::string ExternalContentsPath; 397 NameKind UseName; 398 public: 399 FileEntry(StringRef Name, StringRef ExternalContentsPath, NameKind UseName) 400 : Entry(EK_File, Name), ExternalContentsPath(ExternalContentsPath), 401 UseName(UseName) {} 402 StringRef getExternalContentsPath() const { return ExternalContentsPath; } 403 /// \brief whether to use the external path as the name for this file. 404 bool useExternalName(bool GlobalUseExternalName) const { 405 return UseName == NK_NotSet ? GlobalUseExternalName 406 : (UseName == NK_External); 407 } 408 static bool classof(const Entry *E) { return E->getKind() == EK_File; } 409 }; 410 411 class VFSFromYAML; 412 413 class VFSFromYamlDirIterImpl : public clang::vfs::detail::DirIterImpl { 414 std::string Dir; 415 VFSFromYAML &FS; 416 DirectoryEntry::iterator Current, End; 417 public: 418 VFSFromYamlDirIterImpl(const Twine &Path, VFSFromYAML &FS, 419 DirectoryEntry::iterator Begin, 420 DirectoryEntry::iterator End, std::error_code &EC); 421 std::error_code increment() override; 422 }; 423 424 /// \brief A virtual file system parsed from a YAML file. 425 /// 426 /// Currently, this class allows creating virtual directories and mapping 427 /// virtual file paths to existing external files, available in \c ExternalFS. 428 /// 429 /// The basic structure of the parsed file is: 430 /// \verbatim 431 /// { 432 /// 'version': <version number>, 433 /// <optional configuration> 434 /// 'roots': [ 435 /// <directory entries> 436 /// ] 437 /// } 438 /// \endverbatim 439 /// 440 /// All configuration options are optional. 441 /// 'case-sensitive': <boolean, default=true> 442 /// 'use-external-names': <boolean, default=true> 443 /// 444 /// Virtual directories are represented as 445 /// \verbatim 446 /// { 447 /// 'type': 'directory', 448 /// 'name': <string>, 449 /// 'contents': [ <file or directory entries> ] 450 /// } 451 /// \endverbatim 452 /// 453 /// The default attributes for virtual directories are: 454 /// \verbatim 455 /// MTime = now() when created 456 /// Perms = 0777 457 /// User = Group = 0 458 /// Size = 0 459 /// UniqueID = unspecified unique value 460 /// \endverbatim 461 /// 462 /// Re-mapped files are represented as 463 /// \verbatim 464 /// { 465 /// 'type': 'file', 466 /// 'name': <string>, 467 /// 'use-external-name': <boolean> # Optional 468 /// 'external-contents': <path to external file>) 469 /// } 470 /// \endverbatim 471 /// 472 /// and inherit their attributes from the external contents. 473 /// 474 /// In both cases, the 'name' field may contain multiple path components (e.g. 475 /// /path/to/file). However, any directory that contains more than one child 476 /// must be uniquely represented by a directory entry. 477 class VFSFromYAML : public vfs::FileSystem { 478 std::vector<Entry *> Roots; ///< The root(s) of the virtual file system. 479 /// \brief The file system to use for external references. 480 IntrusiveRefCntPtr<FileSystem> ExternalFS; 481 482 /// @name Configuration 483 /// @{ 484 485 /// \brief Whether to perform case-sensitive comparisons. 486 /// 487 /// Currently, case-insensitive matching only works correctly with ASCII. 488 bool CaseSensitive; 489 490 /// \brief Whether to use to use the value of 'external-contents' for the 491 /// names of files. This global value is overridable on a per-file basis. 492 bool UseExternalNames; 493 /// @} 494 495 friend class VFSFromYAMLParser; 496 497 private: 498 VFSFromYAML(IntrusiveRefCntPtr<FileSystem> ExternalFS) 499 : ExternalFS(ExternalFS), CaseSensitive(true), UseExternalNames(true) {} 500 501 /// \brief Looks up \p Path in \c Roots. 502 ErrorOr<Entry *> lookupPath(const Twine &Path); 503 504 /// \brief Looks up the path <tt>[Start, End)</tt> in \p From, possibly 505 /// recursing into the contents of \p From if it is a directory. 506 ErrorOr<Entry *> lookupPath(sys::path::const_iterator Start, 507 sys::path::const_iterator End, Entry *From); 508 509 /// \brief Get the status of a given an \c Entry. 510 ErrorOr<Status> status(const Twine &Path, Entry *E); 511 512 public: 513 ~VFSFromYAML(); 514 515 /// \brief Parses \p Buffer, which is expected to be in YAML format and 516 /// returns a virtual file system representing its contents. 517 /// 518 /// Takes ownership of \p Buffer. 519 static VFSFromYAML *create(MemoryBuffer *Buffer, 520 SourceMgr::DiagHandlerTy DiagHandler, 521 void *DiagContext, 522 IntrusiveRefCntPtr<FileSystem> ExternalFS); 523 524 ErrorOr<Status> status(const Twine &Path) override; 525 std::error_code openFileForRead(const Twine &Path, 526 std::unique_ptr<File> &Result) override; 527 528 directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override{ 529 ErrorOr<Entry *> E = lookupPath(Dir); 530 if (!E) { 531 EC = E.getError(); 532 return directory_iterator(); 533 } 534 ErrorOr<Status> S = status(Dir, *E); 535 if (!S) { 536 EC = S.getError(); 537 return directory_iterator(); 538 } 539 if (!S->isDirectory()) { 540 EC = std::error_code(static_cast<int>(errc::not_a_directory), 541 std::system_category()); 542 return directory_iterator(); 543 } 544 545 DirectoryEntry *D = cast<DirectoryEntry>(*E); 546 return directory_iterator(std::make_shared<VFSFromYamlDirIterImpl>(Dir, 547 *this, D->contents_begin(), D->contents_end(), EC)); 548 } 549 }; 550 551 /// \brief A helper class to hold the common YAML parsing state. 552 class VFSFromYAMLParser { 553 yaml::Stream &Stream; 554 555 void error(yaml::Node *N, const Twine &Msg) { 556 Stream.printError(N, Msg); 557 } 558 559 // false on error 560 bool parseScalarString(yaml::Node *N, StringRef &Result, 561 SmallVectorImpl<char> &Storage) { 562 yaml::ScalarNode *S = dyn_cast<yaml::ScalarNode>(N); 563 if (!S) { 564 error(N, "expected string"); 565 return false; 566 } 567 Result = S->getValue(Storage); 568 return true; 569 } 570 571 // false on error 572 bool parseScalarBool(yaml::Node *N, bool &Result) { 573 SmallString<5> Storage; 574 StringRef Value; 575 if (!parseScalarString(N, Value, Storage)) 576 return false; 577 578 if (Value.equals_lower("true") || Value.equals_lower("on") || 579 Value.equals_lower("yes") || Value == "1") { 580 Result = true; 581 return true; 582 } else if (Value.equals_lower("false") || Value.equals_lower("off") || 583 Value.equals_lower("no") || Value == "0") { 584 Result = false; 585 return true; 586 } 587 588 error(N, "expected boolean value"); 589 return false; 590 } 591 592 struct KeyStatus { 593 KeyStatus(bool Required=false) : Required(Required), Seen(false) {} 594 bool Required; 595 bool Seen; 596 }; 597 typedef std::pair<StringRef, KeyStatus> KeyStatusPair; 598 599 // false on error 600 bool checkDuplicateOrUnknownKey(yaml::Node *KeyNode, StringRef Key, 601 DenseMap<StringRef, KeyStatus> &Keys) { 602 if (!Keys.count(Key)) { 603 error(KeyNode, "unknown key"); 604 return false; 605 } 606 KeyStatus &S = Keys[Key]; 607 if (S.Seen) { 608 error(KeyNode, Twine("duplicate key '") + Key + "'"); 609 return false; 610 } 611 S.Seen = true; 612 return true; 613 } 614 615 // false on error 616 bool checkMissingKeys(yaml::Node *Obj, DenseMap<StringRef, KeyStatus> &Keys) { 617 for (DenseMap<StringRef, KeyStatus>::iterator I = Keys.begin(), 618 E = Keys.end(); 619 I != E; ++I) { 620 if (I->second.Required && !I->second.Seen) { 621 error(Obj, Twine("missing key '") + I->first + "'"); 622 return false; 623 } 624 } 625 return true; 626 } 627 628 Entry *parseEntry(yaml::Node *N) { 629 yaml::MappingNode *M = dyn_cast<yaml::MappingNode>(N); 630 if (!M) { 631 error(N, "expected mapping node for file or directory entry"); 632 return nullptr; 633 } 634 635 KeyStatusPair Fields[] = { 636 KeyStatusPair("name", true), 637 KeyStatusPair("type", true), 638 KeyStatusPair("contents", false), 639 KeyStatusPair("external-contents", false), 640 KeyStatusPair("use-external-name", false), 641 }; 642 643 DenseMap<StringRef, KeyStatus> Keys( 644 &Fields[0], Fields + sizeof(Fields)/sizeof(Fields[0])); 645 646 bool HasContents = false; // external or otherwise 647 std::vector<Entry *> EntryArrayContents; 648 std::string ExternalContentsPath; 649 std::string Name; 650 FileEntry::NameKind UseExternalName = FileEntry::NK_NotSet; 651 EntryKind Kind; 652 653 for (yaml::MappingNode::iterator I = M->begin(), E = M->end(); I != E; 654 ++I) { 655 StringRef Key; 656 // Reuse the buffer for key and value, since we don't look at key after 657 // parsing value. 658 SmallString<256> Buffer; 659 if (!parseScalarString(I->getKey(), Key, Buffer)) 660 return nullptr; 661 662 if (!checkDuplicateOrUnknownKey(I->getKey(), Key, Keys)) 663 return nullptr; 664 665 StringRef Value; 666 if (Key == "name") { 667 if (!parseScalarString(I->getValue(), Value, Buffer)) 668 return nullptr; 669 Name = Value; 670 } else if (Key == "type") { 671 if (!parseScalarString(I->getValue(), Value, Buffer)) 672 return nullptr; 673 if (Value == "file") 674 Kind = EK_File; 675 else if (Value == "directory") 676 Kind = EK_Directory; 677 else { 678 error(I->getValue(), "unknown value for 'type'"); 679 return nullptr; 680 } 681 } else if (Key == "contents") { 682 if (HasContents) { 683 error(I->getKey(), 684 "entry already has 'contents' or 'external-contents'"); 685 return nullptr; 686 } 687 HasContents = true; 688 yaml::SequenceNode *Contents = 689 dyn_cast<yaml::SequenceNode>(I->getValue()); 690 if (!Contents) { 691 // FIXME: this is only for directories, what about files? 692 error(I->getValue(), "expected array"); 693 return nullptr; 694 } 695 696 for (yaml::SequenceNode::iterator I = Contents->begin(), 697 E = Contents->end(); 698 I != E; ++I) { 699 if (Entry *E = parseEntry(&*I)) 700 EntryArrayContents.push_back(E); 701 else 702 return nullptr; 703 } 704 } else if (Key == "external-contents") { 705 if (HasContents) { 706 error(I->getKey(), 707 "entry already has 'contents' or 'external-contents'"); 708 return nullptr; 709 } 710 HasContents = true; 711 if (!parseScalarString(I->getValue(), Value, Buffer)) 712 return nullptr; 713 ExternalContentsPath = Value; 714 } else if (Key == "use-external-name") { 715 bool Val; 716 if (!parseScalarBool(I->getValue(), Val)) 717 return nullptr; 718 UseExternalName = Val ? FileEntry::NK_External : FileEntry::NK_Virtual; 719 } else { 720 llvm_unreachable("key missing from Keys"); 721 } 722 } 723 724 if (Stream.failed()) 725 return nullptr; 726 727 // check for missing keys 728 if (!HasContents) { 729 error(N, "missing key 'contents' or 'external-contents'"); 730 return nullptr; 731 } 732 if (!checkMissingKeys(N, Keys)) 733 return nullptr; 734 735 // check invalid configuration 736 if (Kind == EK_Directory && UseExternalName != FileEntry::NK_NotSet) { 737 error(N, "'use-external-name' is not supported for directories"); 738 return nullptr; 739 } 740 741 // Remove trailing slash(es), being careful not to remove the root path 742 StringRef Trimmed(Name); 743 size_t RootPathLen = sys::path::root_path(Trimmed).size(); 744 while (Trimmed.size() > RootPathLen && 745 sys::path::is_separator(Trimmed.back())) 746 Trimmed = Trimmed.slice(0, Trimmed.size()-1); 747 // Get the last component 748 StringRef LastComponent = sys::path::filename(Trimmed); 749 750 Entry *Result = nullptr; 751 switch (Kind) { 752 case EK_File: 753 Result = new FileEntry(LastComponent, std::move(ExternalContentsPath), 754 UseExternalName); 755 break; 756 case EK_Directory: 757 Result = new DirectoryEntry(LastComponent, std::move(EntryArrayContents), 758 Status("", "", getNextVirtualUniqueID(), sys::TimeValue::now(), 0, 0, 759 0, file_type::directory_file, sys::fs::all_all)); 760 break; 761 } 762 763 StringRef Parent = sys::path::parent_path(Trimmed); 764 if (Parent.empty()) 765 return Result; 766 767 // if 'name' contains multiple components, create implicit directory entries 768 for (sys::path::reverse_iterator I = sys::path::rbegin(Parent), 769 E = sys::path::rend(Parent); 770 I != E; ++I) { 771 Result = new DirectoryEntry(*I, llvm::makeArrayRef(Result), 772 Status("", "", getNextVirtualUniqueID(), sys::TimeValue::now(), 0, 0, 773 0, file_type::directory_file, sys::fs::all_all)); 774 } 775 return Result; 776 } 777 778 public: 779 VFSFromYAMLParser(yaml::Stream &S) : Stream(S) {} 780 781 // false on error 782 bool parse(yaml::Node *Root, VFSFromYAML *FS) { 783 yaml::MappingNode *Top = dyn_cast<yaml::MappingNode>(Root); 784 if (!Top) { 785 error(Root, "expected mapping node"); 786 return false; 787 } 788 789 KeyStatusPair Fields[] = { 790 KeyStatusPair("version", true), 791 KeyStatusPair("case-sensitive", false), 792 KeyStatusPair("use-external-names", false), 793 KeyStatusPair("roots", true), 794 }; 795 796 DenseMap<StringRef, KeyStatus> Keys( 797 &Fields[0], Fields + sizeof(Fields)/sizeof(Fields[0])); 798 799 // Parse configuration and 'roots' 800 for (yaml::MappingNode::iterator I = Top->begin(), E = Top->end(); I != E; 801 ++I) { 802 SmallString<10> KeyBuffer; 803 StringRef Key; 804 if (!parseScalarString(I->getKey(), Key, KeyBuffer)) 805 return false; 806 807 if (!checkDuplicateOrUnknownKey(I->getKey(), Key, Keys)) 808 return false; 809 810 if (Key == "roots") { 811 yaml::SequenceNode *Roots = dyn_cast<yaml::SequenceNode>(I->getValue()); 812 if (!Roots) { 813 error(I->getValue(), "expected array"); 814 return false; 815 } 816 817 for (yaml::SequenceNode::iterator I = Roots->begin(), E = Roots->end(); 818 I != E; ++I) { 819 if (Entry *E = parseEntry(&*I)) 820 FS->Roots.push_back(E); 821 else 822 return false; 823 } 824 } else if (Key == "version") { 825 StringRef VersionString; 826 SmallString<4> Storage; 827 if (!parseScalarString(I->getValue(), VersionString, Storage)) 828 return false; 829 int Version; 830 if (VersionString.getAsInteger<int>(10, Version)) { 831 error(I->getValue(), "expected integer"); 832 return false; 833 } 834 if (Version < 0) { 835 error(I->getValue(), "invalid version number"); 836 return false; 837 } 838 if (Version != 0) { 839 error(I->getValue(), "version mismatch, expected 0"); 840 return false; 841 } 842 } else if (Key == "case-sensitive") { 843 if (!parseScalarBool(I->getValue(), FS->CaseSensitive)) 844 return false; 845 } else if (Key == "use-external-names") { 846 if (!parseScalarBool(I->getValue(), FS->UseExternalNames)) 847 return false; 848 } else { 849 llvm_unreachable("key missing from Keys"); 850 } 851 } 852 853 if (Stream.failed()) 854 return false; 855 856 if (!checkMissingKeys(Top, Keys)) 857 return false; 858 return true; 859 } 860 }; 861 } // end of anonymous namespace 862 863 Entry::~Entry() {} 864 DirectoryEntry::~DirectoryEntry() { llvm::DeleteContainerPointers(Contents); } 865 866 VFSFromYAML::~VFSFromYAML() { llvm::DeleteContainerPointers(Roots); } 867 868 VFSFromYAML *VFSFromYAML::create(MemoryBuffer *Buffer, 869 SourceMgr::DiagHandlerTy DiagHandler, 870 void *DiagContext, 871 IntrusiveRefCntPtr<FileSystem> ExternalFS) { 872 873 SourceMgr SM; 874 yaml::Stream Stream(Buffer, SM); 875 876 SM.setDiagHandler(DiagHandler, DiagContext); 877 yaml::document_iterator DI = Stream.begin(); 878 yaml::Node *Root = DI->getRoot(); 879 if (DI == Stream.end() || !Root) { 880 SM.PrintMessage(SMLoc(), SourceMgr::DK_Error, "expected root node"); 881 return nullptr; 882 } 883 884 VFSFromYAMLParser P(Stream); 885 886 std::unique_ptr<VFSFromYAML> FS(new VFSFromYAML(ExternalFS)); 887 if (!P.parse(Root, FS.get())) 888 return nullptr; 889 890 return FS.release(); 891 } 892 893 ErrorOr<Entry *> VFSFromYAML::lookupPath(const Twine &Path_) { 894 SmallString<256> Path; 895 Path_.toVector(Path); 896 897 // Handle relative paths 898 if (std::error_code EC = sys::fs::make_absolute(Path)) 899 return EC; 900 901 if (Path.empty()) 902 return make_error_code(llvm::errc::invalid_argument); 903 904 sys::path::const_iterator Start = sys::path::begin(Path); 905 sys::path::const_iterator End = sys::path::end(Path); 906 for (std::vector<Entry *>::iterator I = Roots.begin(), E = Roots.end(); 907 I != E; ++I) { 908 ErrorOr<Entry *> Result = lookupPath(Start, End, *I); 909 if (Result || Result.getError() != llvm::errc::no_such_file_or_directory) 910 return Result; 911 } 912 return make_error_code(llvm::errc::no_such_file_or_directory); 913 } 914 915 ErrorOr<Entry *> VFSFromYAML::lookupPath(sys::path::const_iterator Start, 916 sys::path::const_iterator End, 917 Entry *From) { 918 if (Start->equals(".")) 919 ++Start; 920 921 // FIXME: handle .. 922 if (CaseSensitive ? !Start->equals(From->getName()) 923 : !Start->equals_lower(From->getName())) 924 // failure to match 925 return make_error_code(llvm::errc::no_such_file_or_directory); 926 927 ++Start; 928 929 if (Start == End) { 930 // Match! 931 return From; 932 } 933 934 DirectoryEntry *DE = dyn_cast<DirectoryEntry>(From); 935 if (!DE) 936 return make_error_code(llvm::errc::not_a_directory); 937 938 for (DirectoryEntry::iterator I = DE->contents_begin(), 939 E = DE->contents_end(); 940 I != E; ++I) { 941 ErrorOr<Entry *> Result = lookupPath(Start, End, *I); 942 if (Result || Result.getError() != llvm::errc::no_such_file_or_directory) 943 return Result; 944 } 945 return make_error_code(llvm::errc::no_such_file_or_directory); 946 } 947 948 ErrorOr<Status> VFSFromYAML::status(const Twine &Path, Entry *E) { 949 assert(E != nullptr); 950 std::string PathStr(Path.str()); 951 if (FileEntry *F = dyn_cast<FileEntry>(E)) { 952 ErrorOr<Status> S = ExternalFS->status(F->getExternalContentsPath()); 953 assert(!S || S->getName() == F->getExternalContentsPath()); 954 if (S && !F->useExternalName(UseExternalNames)) 955 S->setName(PathStr); 956 if (S) 957 S->IsVFSMapped = true; 958 return S; 959 } else { // directory 960 DirectoryEntry *DE = cast<DirectoryEntry>(E); 961 Status S = DE->getStatus(); 962 S.setName(PathStr); 963 return S; 964 } 965 } 966 967 ErrorOr<Status> VFSFromYAML::status(const Twine &Path) { 968 ErrorOr<Entry *> Result = lookupPath(Path); 969 if (!Result) 970 return Result.getError(); 971 return status(Path, *Result); 972 } 973 974 std::error_code 975 VFSFromYAML::openFileForRead(const Twine &Path, 976 std::unique_ptr<vfs::File> &Result) { 977 ErrorOr<Entry *> E = lookupPath(Path); 978 if (!E) 979 return E.getError(); 980 981 FileEntry *F = dyn_cast<FileEntry>(*E); 982 if (!F) // FIXME: errc::not_a_file? 983 return make_error_code(llvm::errc::invalid_argument); 984 985 if (std::error_code EC = 986 ExternalFS->openFileForRead(F->getExternalContentsPath(), Result)) 987 return EC; 988 989 if (!F->useExternalName(UseExternalNames)) 990 Result->setName(Path.str()); 991 992 return std::error_code(); 993 } 994 995 IntrusiveRefCntPtr<FileSystem> 996 vfs::getVFSFromYAML(MemoryBuffer *Buffer, SourceMgr::DiagHandlerTy DiagHandler, 997 void *DiagContext, 998 IntrusiveRefCntPtr<FileSystem> ExternalFS) { 999 return VFSFromYAML::create(Buffer, DiagHandler, DiagContext, ExternalFS); 1000 } 1001 1002 UniqueID vfs::getNextVirtualUniqueID() { 1003 static std::atomic<unsigned> UID; 1004 unsigned ID = ++UID; 1005 // The following assumes that uint64_t max will never collide with a real 1006 // dev_t value from the OS. 1007 return UniqueID(std::numeric_limits<uint64_t>::max(), ID); 1008 } 1009 1010 #ifndef NDEBUG 1011 static bool pathHasTraversal(StringRef Path) { 1012 using namespace llvm::sys; 1013 for (StringRef Comp : llvm::make_range(path::begin(Path), path::end(Path))) 1014 if (Comp == "." || Comp == "..") 1015 return true; 1016 return false; 1017 } 1018 #endif 1019 1020 void YAMLVFSWriter::addFileMapping(StringRef VirtualPath, StringRef RealPath) { 1021 assert(sys::path::is_absolute(VirtualPath) && "virtual path not absolute"); 1022 assert(sys::path::is_absolute(RealPath) && "real path not absolute"); 1023 assert(!pathHasTraversal(VirtualPath) && "path traversal is not supported"); 1024 Mappings.emplace_back(VirtualPath, RealPath); 1025 } 1026 1027 namespace { 1028 class JSONWriter { 1029 llvm::raw_ostream &OS; 1030 SmallVector<StringRef, 16> DirStack; 1031 inline unsigned getDirIndent() { return 4 * DirStack.size(); } 1032 inline unsigned getFileIndent() { return 4 * (DirStack.size() + 1); } 1033 bool containedIn(StringRef Parent, StringRef Path); 1034 StringRef containedPart(StringRef Parent, StringRef Path); 1035 void startDirectory(StringRef Path); 1036 void endDirectory(); 1037 void writeEntry(StringRef VPath, StringRef RPath); 1038 1039 public: 1040 JSONWriter(llvm::raw_ostream &OS) : OS(OS) {} 1041 void write(ArrayRef<YAMLVFSEntry> Entries, Optional<bool> IsCaseSensitive); 1042 }; 1043 } 1044 1045 bool JSONWriter::containedIn(StringRef Parent, StringRef Path) { 1046 using namespace llvm::sys; 1047 // Compare each path component. 1048 auto IParent = path::begin(Parent), EParent = path::end(Parent); 1049 for (auto IChild = path::begin(Path), EChild = path::end(Path); 1050 IParent != EParent && IChild != EChild; ++IParent, ++IChild) { 1051 if (*IParent != *IChild) 1052 return false; 1053 } 1054 // Have we exhausted the parent path? 1055 return IParent == EParent; 1056 } 1057 1058 StringRef JSONWriter::containedPart(StringRef Parent, StringRef Path) { 1059 assert(!Parent.empty()); 1060 assert(containedIn(Parent, Path)); 1061 return Path.slice(Parent.size() + 1, StringRef::npos); 1062 } 1063 1064 void JSONWriter::startDirectory(StringRef Path) { 1065 StringRef Name = 1066 DirStack.empty() ? Path : containedPart(DirStack.back(), Path); 1067 DirStack.push_back(Path); 1068 unsigned Indent = getDirIndent(); 1069 OS.indent(Indent) << "{\n"; 1070 OS.indent(Indent + 2) << "'type': 'directory',\n"; 1071 OS.indent(Indent + 2) << "'name': \"" << llvm::yaml::escape(Name) << "\",\n"; 1072 OS.indent(Indent + 2) << "'contents': [\n"; 1073 } 1074 1075 void JSONWriter::endDirectory() { 1076 unsigned Indent = getDirIndent(); 1077 OS.indent(Indent + 2) << "]\n"; 1078 OS.indent(Indent) << "}"; 1079 1080 DirStack.pop_back(); 1081 } 1082 1083 void JSONWriter::writeEntry(StringRef VPath, StringRef RPath) { 1084 unsigned Indent = getFileIndent(); 1085 OS.indent(Indent) << "{\n"; 1086 OS.indent(Indent + 2) << "'type': 'file',\n"; 1087 OS.indent(Indent + 2) << "'name': \"" << llvm::yaml::escape(VPath) << "\",\n"; 1088 OS.indent(Indent + 2) << "'external-contents': \"" 1089 << llvm::yaml::escape(RPath) << "\"\n"; 1090 OS.indent(Indent) << "}"; 1091 } 1092 1093 void JSONWriter::write(ArrayRef<YAMLVFSEntry> Entries, 1094 Optional<bool> IsCaseSensitive) { 1095 using namespace llvm::sys; 1096 1097 OS << "{\n" 1098 " 'version': 0,\n"; 1099 if (IsCaseSensitive.hasValue()) 1100 OS << " 'case-sensitive': '" 1101 << (IsCaseSensitive.getValue() ? "true" : "false") << "',\n"; 1102 OS << " 'roots': [\n"; 1103 1104 if (Entries.empty()) 1105 return; 1106 1107 const YAMLVFSEntry &Entry = Entries.front(); 1108 startDirectory(path::parent_path(Entry.VPath)); 1109 writeEntry(path::filename(Entry.VPath), Entry.RPath); 1110 1111 for (const auto &Entry : Entries.slice(1)) { 1112 StringRef Dir = path::parent_path(Entry.VPath); 1113 if (Dir == DirStack.back()) 1114 OS << ",\n"; 1115 else { 1116 while (!DirStack.empty() && !containedIn(DirStack.back(), Dir)) { 1117 OS << "\n"; 1118 endDirectory(); 1119 } 1120 OS << ",\n"; 1121 startDirectory(Dir); 1122 } 1123 writeEntry(path::filename(Entry.VPath), Entry.RPath); 1124 } 1125 1126 while (!DirStack.empty()) { 1127 OS << "\n"; 1128 endDirectory(); 1129 } 1130 1131 OS << "\n" 1132 << " ]\n" 1133 << "}\n"; 1134 } 1135 1136 void YAMLVFSWriter::write(llvm::raw_ostream &OS) { 1137 std::sort(Mappings.begin(), Mappings.end(), 1138 [](const YAMLVFSEntry &LHS, const YAMLVFSEntry &RHS) { 1139 return LHS.VPath < RHS.VPath; 1140 }); 1141 1142 JSONWriter(OS).write(Mappings, IsCaseSensitive); 1143 } 1144 1145 VFSFromYamlDirIterImpl::VFSFromYamlDirIterImpl(const Twine &_Path, 1146 VFSFromYAML &FS, 1147 DirectoryEntry::iterator Begin, 1148 DirectoryEntry::iterator End, 1149 std::error_code &EC) 1150 : Dir(_Path.str()), FS(FS), Current(Begin), End(End) { 1151 if (Current != End) { 1152 SmallString<128> PathStr(Dir); 1153 llvm::sys::path::append(PathStr, (*Current)->getName()); 1154 llvm::ErrorOr<vfs::Status> S = FS.status(PathStr.str()); 1155 if (S) 1156 CurrentEntry = *S; 1157 else 1158 EC = S.getError(); 1159 } 1160 } 1161 1162 std::error_code VFSFromYamlDirIterImpl::increment() { 1163 assert(Current != End && "cannot iterate past end"); 1164 if (++Current != End) { 1165 SmallString<128> PathStr(Dir); 1166 llvm::sys::path::append(PathStr, (*Current)->getName()); 1167 llvm::ErrorOr<vfs::Status> S = FS.status(PathStr.str()); 1168 if (!S) 1169 return S.getError(); 1170 CurrentEntry = *S; 1171 } else { 1172 CurrentEntry = Status(); 1173 } 1174 return std::error_code(); 1175 } 1176 1177 vfs::recursive_directory_iterator::recursive_directory_iterator(FileSystem &FS_, 1178 const Twine &Path, 1179 std::error_code &EC) 1180 : FS(&FS_) { 1181 directory_iterator I = FS->dir_begin(Path, EC); 1182 if (!EC && I != directory_iterator()) { 1183 State = std::make_shared<IterState>(); 1184 State->push(I); 1185 } 1186 } 1187 1188 vfs::recursive_directory_iterator & 1189 recursive_directory_iterator::increment(std::error_code &EC) { 1190 assert(FS && State && !State->empty() && "incrementing past end"); 1191 assert(State->top()->isStatusKnown() && "non-canonical end iterator"); 1192 vfs::directory_iterator End; 1193 if (State->top()->isDirectory()) { 1194 vfs::directory_iterator I = FS->dir_begin(State->top()->getName(), EC); 1195 if (EC) 1196 return *this; 1197 if (I != End) { 1198 State->push(I); 1199 return *this; 1200 } 1201 } 1202 1203 while (!State->empty() && State->top().increment(EC) == End) 1204 State->pop(); 1205 1206 if (State->empty()) 1207 State.reset(); // end iterator 1208 1209 return *this; 1210 } 1211