1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #include <sys/stat.h> 17 #include <deque> 18 #include <utility> 19 #include <vector> 20 #if defined(__APPLE__) 21 #include <mach-o/dyld.h> 22 #endif 23 #if defined(__FreeBSD__) 24 #include <sys/sysctl.h> 25 #include <sys/types.h> 26 #endif 27 #if defined(PLATFORM_WINDOWS) 28 #include <windows.h> 29 #include "tensorflow/core/platform/windows/wide_char.h" 30 #define PATH_MAX MAX_PATH 31 #else 32 #include <fcntl.h> 33 #include <string.h> 34 #include <sys/types.h> 35 #include <unistd.h> 36 #endif 37 38 #include "tensorflow/core/lib/core/errors.h" 39 #include "tensorflow/core/lib/gtl/stl_util.h" 40 #include "tensorflow/core/lib/io/path.h" 41 #include "tensorflow/core/lib/strings/stringprintf.h" 42 #include "tensorflow/core/platform/env.h" 43 #include "tensorflow/core/platform/env_time.h" 44 #include "tensorflow/core/platform/host_info.h" 45 #include "tensorflow/core/platform/protobuf.h" 46 47 namespace tensorflow { 48 49 // 128KB copy buffer 50 constexpr size_t kCopyFileBufferSize = 128 * 1024; 51 52 class FileSystemRegistryImpl : public FileSystemRegistry { 53 public: 54 Status Register(const string& scheme, Factory factory) override; 55 FileSystem* Lookup(const string& scheme) override; 56 Status GetRegisteredFileSystemSchemes(std::vector<string>* schemes) override; 57 58 private: 59 mutable mutex mu_; 60 mutable std::unordered_map<string, std::unique_ptr<FileSystem>> registry_ 61 GUARDED_BY(mu_); 62 }; 63 64 Status FileSystemRegistryImpl::Register(const string& scheme, 65 FileSystemRegistry::Factory factory) { 66 mutex_lock lock(mu_); 67 if (!registry_.emplace(string(scheme), std::unique_ptr<FileSystem>(factory())) 68 .second) { 69 return errors::AlreadyExists("File factory for ", scheme, 70 " already registered"); 71 } 72 return Status::OK(); 73 } 74 75 FileSystem* FileSystemRegistryImpl::Lookup(const string& scheme) { 76 mutex_lock lock(mu_); 77 const auto found = registry_.find(scheme); 78 if (found == registry_.end()) { 79 return nullptr; 80 } 81 return found->second.get(); 82 } 83 84 Status FileSystemRegistryImpl::GetRegisteredFileSystemSchemes( 85 std::vector<string>* schemes) { 86 mutex_lock lock(mu_); 87 for (const auto& e : registry_) { 88 schemes->push_back(e.first); 89 } 90 return Status::OK(); 91 } 92 93 Env::Env() : file_system_registry_(new FileSystemRegistryImpl) {} 94 95 Status Env::GetFileSystemForFile(const string& fname, FileSystem** result) { 96 StringPiece scheme, host, path; 97 io::ParseURI(fname, &scheme, &host, &path); 98 FileSystem* file_system = file_system_registry_->Lookup(string(scheme)); 99 if (!file_system) { 100 if (scheme.empty()) { 101 scheme = "[local]"; 102 } 103 104 return errors::Unimplemented("File system scheme '", scheme, 105 "' not implemented (file: '", fname, "')"); 106 } 107 *result = file_system; 108 return Status::OK(); 109 } 110 111 Status Env::GetRegisteredFileSystemSchemes(std::vector<string>* schemes) { 112 return file_system_registry_->GetRegisteredFileSystemSchemes(schemes); 113 } 114 115 Status Env::RegisterFileSystem(const string& scheme, 116 FileSystemRegistry::Factory factory) { 117 return file_system_registry_->Register(scheme, std::move(factory)); 118 } 119 120 Status Env::FlushFileSystemCaches() { 121 std::vector<string> schemes; 122 TF_RETURN_IF_ERROR(GetRegisteredFileSystemSchemes(&schemes)); 123 for (const string& scheme : schemes) { 124 FileSystem* fs = nullptr; 125 TF_RETURN_IF_ERROR( 126 GetFileSystemForFile(io::CreateURI(scheme, "", ""), &fs)); 127 fs->FlushCaches(); 128 } 129 return Status::OK(); 130 } 131 132 Status Env::NewRandomAccessFile(const string& fname, 133 std::unique_ptr<RandomAccessFile>* result) { 134 FileSystem* fs; 135 TF_RETURN_IF_ERROR(GetFileSystemForFile(fname, &fs)); 136 return fs->NewRandomAccessFile(fname, result); 137 } 138 139 Status Env::NewReadOnlyMemoryRegionFromFile( 140 const string& fname, std::unique_ptr<ReadOnlyMemoryRegion>* result) { 141 FileSystem* fs; 142 TF_RETURN_IF_ERROR(GetFileSystemForFile(fname, &fs)); 143 return fs->NewReadOnlyMemoryRegionFromFile(fname, result); 144 } 145 146 Status Env::NewWritableFile(const string& fname, 147 std::unique_ptr<WritableFile>* result) { 148 FileSystem* fs; 149 TF_RETURN_IF_ERROR(GetFileSystemForFile(fname, &fs)); 150 return fs->NewWritableFile(fname, result); 151 } 152 153 Status Env::NewAppendableFile(const string& fname, 154 std::unique_ptr<WritableFile>* result) { 155 FileSystem* fs; 156 TF_RETURN_IF_ERROR(GetFileSystemForFile(fname, &fs)); 157 return fs->NewAppendableFile(fname, result); 158 } 159 160 Status Env::FileExists(const string& fname) { 161 FileSystem* fs; 162 TF_RETURN_IF_ERROR(GetFileSystemForFile(fname, &fs)); 163 return fs->FileExists(fname); 164 } 165 166 bool Env::FilesExist(const std::vector<string>& files, 167 std::vector<Status>* status) { 168 std::unordered_map<string, std::vector<string>> files_per_fs; 169 for (const auto& file : files) { 170 StringPiece scheme, host, path; 171 io::ParseURI(file, &scheme, &host, &path); 172 files_per_fs[string(scheme)].push_back(file); 173 } 174 175 std::unordered_map<string, Status> per_file_status; 176 bool result = true; 177 for (auto itr : files_per_fs) { 178 FileSystem* file_system = file_system_registry_->Lookup(itr.first); 179 bool fs_result; 180 std::vector<Status> local_status; 181 std::vector<Status>* fs_status = status ? &local_status : nullptr; 182 if (!file_system) { 183 fs_result = false; 184 if (fs_status) { 185 Status s = errors::Unimplemented("File system scheme '", itr.first, 186 "' not implemented"); 187 local_status.resize(itr.second.size(), s); 188 } 189 } else { 190 fs_result = file_system->FilesExist(itr.second, fs_status); 191 } 192 if (fs_status) { 193 result &= fs_result; 194 for (int i = 0; i < itr.second.size(); ++i) { 195 per_file_status[itr.second[i]] = fs_status->at(i); 196 } 197 } else if (!fs_result) { 198 // Return early 199 return false; 200 } 201 } 202 203 if (status) { 204 for (const auto& file : files) { 205 status->push_back(per_file_status[file]); 206 } 207 } 208 209 return result; 210 } 211 212 Status Env::GetChildren(const string& dir, std::vector<string>* result) { 213 FileSystem* fs; 214 TF_RETURN_IF_ERROR(GetFileSystemForFile(dir, &fs)); 215 return fs->GetChildren(dir, result); 216 } 217 218 Status Env::GetMatchingPaths(const string& pattern, 219 std::vector<string>* results) { 220 FileSystem* fs; 221 TF_RETURN_IF_ERROR(GetFileSystemForFile(pattern, &fs)); 222 return fs->GetMatchingPaths(pattern, results); 223 } 224 225 Status Env::DeleteFile(const string& fname) { 226 FileSystem* fs; 227 TF_RETURN_IF_ERROR(GetFileSystemForFile(fname, &fs)); 228 return fs->DeleteFile(fname); 229 } 230 231 Status Env::RecursivelyCreateDir(const string& dirname) { 232 FileSystem* fs; 233 TF_RETURN_IF_ERROR(GetFileSystemForFile(dirname, &fs)); 234 return fs->RecursivelyCreateDir(dirname); 235 } 236 237 Status Env::CreateDir(const string& dirname) { 238 FileSystem* fs; 239 TF_RETURN_IF_ERROR(GetFileSystemForFile(dirname, &fs)); 240 return fs->CreateDir(dirname); 241 } 242 243 Status Env::DeleteDir(const string& dirname) { 244 FileSystem* fs; 245 TF_RETURN_IF_ERROR(GetFileSystemForFile(dirname, &fs)); 246 return fs->DeleteDir(dirname); 247 } 248 249 Status Env::Stat(const string& fname, FileStatistics* stat) { 250 FileSystem* fs; 251 TF_RETURN_IF_ERROR(GetFileSystemForFile(fname, &fs)); 252 return fs->Stat(fname, stat); 253 } 254 255 Status Env::IsDirectory(const string& fname) { 256 FileSystem* fs; 257 TF_RETURN_IF_ERROR(GetFileSystemForFile(fname, &fs)); 258 return fs->IsDirectory(fname); 259 } 260 261 Status Env::DeleteRecursively(const string& dirname, int64* undeleted_files, 262 int64* undeleted_dirs) { 263 FileSystem* fs; 264 TF_RETURN_IF_ERROR(GetFileSystemForFile(dirname, &fs)); 265 return fs->DeleteRecursively(dirname, undeleted_files, undeleted_dirs); 266 } 267 268 Status Env::GetFileSize(const string& fname, uint64* file_size) { 269 FileSystem* fs; 270 TF_RETURN_IF_ERROR(GetFileSystemForFile(fname, &fs)); 271 return fs->GetFileSize(fname, file_size); 272 } 273 274 Status Env::RenameFile(const string& src, const string& target) { 275 FileSystem* src_fs; 276 FileSystem* target_fs; 277 TF_RETURN_IF_ERROR(GetFileSystemForFile(src, &src_fs)); 278 TF_RETURN_IF_ERROR(GetFileSystemForFile(target, &target_fs)); 279 if (src_fs != target_fs) { 280 return errors::Unimplemented("Renaming ", src, " to ", target, 281 " not implemented"); 282 } 283 return src_fs->RenameFile(src, target); 284 } 285 286 Status Env::CopyFile(const string& src, const string& target) { 287 FileSystem* src_fs; 288 FileSystem* target_fs; 289 TF_RETURN_IF_ERROR(GetFileSystemForFile(src, &src_fs)); 290 TF_RETURN_IF_ERROR(GetFileSystemForFile(target, &target_fs)); 291 if (src_fs == target_fs) { 292 return src_fs->CopyFile(src, target); 293 } 294 return FileSystemCopyFile(src_fs, src, target_fs, target); 295 } 296 297 string Env::GetExecutablePath() { 298 char exe_path[PATH_MAX] = {0}; 299 #ifdef __APPLE__ 300 uint32_t buffer_size(0U); 301 _NSGetExecutablePath(nullptr, &buffer_size); 302 char unresolved_path[buffer_size]; 303 _NSGetExecutablePath(unresolved_path, &buffer_size); 304 CHECK(realpath(unresolved_path, exe_path)); 305 #elif defined(__FreeBSD__) 306 int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1}; 307 size_t exe_path_size = PATH_MAX; 308 309 if (sysctl(mib, 4, exe_path, &exe_path_size, NULL, 0) != 0) { 310 // Resolution of path failed 311 return ""; 312 } 313 #elif defined(PLATFORM_WINDOWS) 314 HMODULE hModule = GetModuleHandleW(NULL); 315 WCHAR wc_file_path[MAX_PATH] = {0}; 316 GetModuleFileNameW(hModule, wc_file_path, MAX_PATH); 317 string file_path = WideCharToUtf8(wc_file_path); 318 std::copy(file_path.begin(), file_path.end(), exe_path); 319 #else 320 char buf[PATH_MAX] = {0}; 321 int path_length = readlink("/proc/self/exe", buf, sizeof(buf) - 1); 322 CHECK_NE(-1, path_length); 323 324 if (strstr(buf, "python") != nullptr) { 325 // Discard the path of the python binary, and any flags. 326 int fd = open("/proc/self/cmdline", O_RDONLY); 327 int cmd_length = read(fd, buf, PATH_MAX - 1); 328 CHECK_NE(-1, cmd_length); 329 int token_pos = 0; 330 for (bool token_is_first_or_flag = true; token_is_first_or_flag;) { 331 // Get token length, including null 332 int token_len = strlen(&buf[token_pos]) + 1; 333 token_is_first_or_flag = false; 334 // Check if we can skip without overshooting 335 if (token_pos + token_len < cmd_length) { 336 token_pos += token_len; 337 token_is_first_or_flag = (buf[token_pos] == '-'); // token is a flag 338 } 339 } 340 snprintf(exe_path, sizeof(exe_path), "%s", &buf[token_pos]); 341 } else { 342 snprintf(exe_path, sizeof(exe_path), "%s", buf); 343 } 344 345 #endif 346 // Make sure it's null-terminated: 347 exe_path[sizeof(exe_path) - 1] = 0; 348 349 return exe_path; 350 } 351 352 bool Env::LocalTempFilename(string* filename) { 353 std::vector<string> dirs; 354 GetLocalTempDirectories(&dirs); 355 356 // Try each directory, as they might be full, have inappropriate 357 // permissions or have different problems at times. 358 for (const string& dir : dirs) { 359 *filename = io::JoinPath(dir, "tempfile-"); 360 if (CreateUniqueFileName(filename, "")) { 361 return true; 362 } 363 } 364 return false; 365 } 366 367 bool Env::CreateUniqueFileName(string* prefix, const string& suffix) { 368 int32 tid = GetCurrentThreadId(); 369 #ifdef PLATFORM_WINDOWS 370 int32 pid = static_cast<int32>(GetCurrentProcessId()); 371 #else 372 int32 pid = static_cast<int32>(getpid()); 373 #endif 374 uint64 now_microsec = NowMicros(); 375 376 *prefix += strings::Printf("%s-%x-%d-%llx", port::Hostname().c_str(), tid, 377 pid, now_microsec); 378 379 if (!suffix.empty()) { 380 *prefix += suffix; 381 } 382 if (FileExists(*prefix).ok()) { 383 prefix->clear(); 384 return false; 385 } else { 386 return true; 387 } 388 } 389 390 Thread::~Thread() {} 391 392 EnvWrapper::~EnvWrapper() {} 393 394 Status ReadFileToString(Env* env, const string& fname, string* data) { 395 uint64 file_size; 396 Status s = env->GetFileSize(fname, &file_size); 397 if (!s.ok()) { 398 return s; 399 } 400 std::unique_ptr<RandomAccessFile> file; 401 s = env->NewRandomAccessFile(fname, &file); 402 if (!s.ok()) { 403 return s; 404 } 405 gtl::STLStringResizeUninitialized(data, file_size); 406 char* p = gtl::string_as_array(data); 407 StringPiece result; 408 s = file->Read(0, file_size, &result, p); 409 if (!s.ok()) { 410 data->clear(); 411 } else if (result.size() != file_size) { 412 s = errors::Aborted("File ", fname, " changed while reading: ", file_size, 413 " vs. ", result.size()); 414 data->clear(); 415 } else if (result.data() == p) { 416 // Data is already in the correct location 417 } else { 418 memmove(p, result.data(), result.size()); 419 } 420 return s; 421 } 422 423 Status WriteStringToFile(Env* env, const string& fname, 424 const StringPiece& data) { 425 std::unique_ptr<WritableFile> file; 426 Status s = env->NewWritableFile(fname, &file); 427 if (!s.ok()) { 428 return s; 429 } 430 s = file->Append(data); 431 if (s.ok()) { 432 s = file->Close(); 433 } 434 return s; 435 } 436 437 Status FileSystemCopyFile(FileSystem* src_fs, const string& src, 438 FileSystem* target_fs, const string& target) { 439 std::unique_ptr<RandomAccessFile> src_file; 440 TF_RETURN_IF_ERROR(src_fs->NewRandomAccessFile(src, &src_file)); 441 442 std::unique_ptr<WritableFile> target_file; 443 TF_RETURN_IF_ERROR(target_fs->NewWritableFile(target, &target_file)); 444 445 uint64 offset = 0; 446 std::unique_ptr<char[]> scratch(new char[kCopyFileBufferSize]); 447 Status s = Status::OK(); 448 while (s.ok()) { 449 StringPiece result; 450 s = src_file->Read(offset, kCopyFileBufferSize, &result, scratch.get()); 451 if (!(s.ok() || s.code() == error::OUT_OF_RANGE)) { 452 return s; 453 } 454 TF_RETURN_IF_ERROR(target_file->Append(result)); 455 offset += result.size(); 456 } 457 return target_file->Close(); 458 } 459 460 // A ZeroCopyInputStream on a RandomAccessFile. 461 namespace { 462 class FileStream : public ::tensorflow::protobuf::io::ZeroCopyInputStream { 463 public: 464 explicit FileStream(RandomAccessFile* file) : file_(file), pos_(0) {} 465 466 void BackUp(int count) override { pos_ -= count; } 467 bool Skip(int count) override { 468 pos_ += count; 469 return true; 470 } 471 protobuf_int64 ByteCount() const override { return pos_; } 472 Status status() const { return status_; } 473 474 bool Next(const void** data, int* size) override { 475 StringPiece result; 476 Status s = file_->Read(pos_, kBufSize, &result, scratch_); 477 if (result.empty()) { 478 status_ = s; 479 return false; 480 } 481 pos_ += result.size(); 482 *data = result.data(); 483 *size = result.size(); 484 return true; 485 } 486 487 private: 488 static const int kBufSize = 512 << 10; 489 490 RandomAccessFile* file_; 491 int64 pos_; 492 Status status_; 493 char scratch_[kBufSize]; 494 }; 495 496 } // namespace 497 498 Status WriteBinaryProto(Env* env, const string& fname, 499 const ::tensorflow::protobuf::MessageLite& proto) { 500 string serialized; 501 proto.AppendToString(&serialized); 502 return WriteStringToFile(env, fname, serialized); 503 } 504 505 Status ReadBinaryProto(Env* env, const string& fname, 506 ::tensorflow::protobuf::MessageLite* proto) { 507 std::unique_ptr<RandomAccessFile> file; 508 TF_RETURN_IF_ERROR(env->NewRandomAccessFile(fname, &file)); 509 std::unique_ptr<FileStream> stream(new FileStream(file.get())); 510 511 // TODO(jiayq): the following coded stream is for debugging purposes to allow 512 // one to parse arbitrarily large messages for MessageLite. One most likely 513 // doesn't want to put protobufs larger than 64MB on Android, so we should 514 // eventually remove this and quit loud when a large protobuf is passed in. 515 ::tensorflow::protobuf::io::CodedInputStream coded_stream(stream.get()); 516 // Total bytes hard limit / warning limit are set to 1GB and 512MB 517 // respectively. 518 coded_stream.SetTotalBytesLimit(1024LL << 20, 512LL << 20); 519 520 if (!proto->ParseFromCodedStream(&coded_stream)) { 521 TF_RETURN_IF_ERROR(stream->status()); 522 return errors::DataLoss("Can't parse ", fname, " as binary proto"); 523 } 524 return Status::OK(); 525 } 526 527 Status WriteTextProto(Env* env, const string& fname, 528 const ::tensorflow::protobuf::Message& proto) { 529 #if !defined(TENSORFLOW_LITE_PROTOS) 530 string serialized; 531 if (!::tensorflow::protobuf::TextFormat::PrintToString(proto, &serialized)) { 532 return errors::FailedPrecondition("Unable to convert proto to text."); 533 } 534 return WriteStringToFile(env, fname, serialized); 535 #else 536 return errors::Unimplemented("Can't write text protos with protolite."); 537 #endif 538 } 539 540 Status ReadTextProto(Env* env, const string& fname, 541 ::tensorflow::protobuf::Message* proto) { 542 #if !defined(TENSORFLOW_LITE_PROTOS) 543 std::unique_ptr<RandomAccessFile> file; 544 TF_RETURN_IF_ERROR(env->NewRandomAccessFile(fname, &file)); 545 std::unique_ptr<FileStream> stream(new FileStream(file.get())); 546 547 if (!::tensorflow::protobuf::TextFormat::Parse(stream.get(), proto)) { 548 TF_RETURN_IF_ERROR(stream->status()); 549 return errors::DataLoss("Can't parse ", fname, " as text proto"); 550 } 551 return Status::OK(); 552 #else 553 return errors::Unimplemented("Can't parse text protos with protolite."); 554 #endif 555 } 556 557 } // namespace tensorflow 558