1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #ifndef TENSORFLOW_CORE_PLATFORM_ENV_H_ 17 #define TENSORFLOW_CORE_PLATFORM_ENV_H_ 18 19 #include <stdint.h> 20 #include <memory> 21 #include <string> 22 #include <unordered_map> 23 #include <vector> 24 #include "tensorflow/core/lib/core/errors.h" 25 #include "tensorflow/core/lib/core/status.h" 26 #include "tensorflow/core/lib/core/stringpiece.h" 27 #include "tensorflow/core/platform/env_time.h" 28 #include "tensorflow/core/platform/file_system.h" 29 #include "tensorflow/core/platform/macros.h" 30 #include "tensorflow/core/platform/mutex.h" 31 #include "tensorflow/core/platform/protobuf.h" 32 #include "tensorflow/core/platform/types.h" 33 34 namespace tensorflow { 35 36 class Thread; 37 struct ThreadOptions; 38 39 /// \brief An interface used by the tensorflow implementation to 40 /// access operating system functionality like the filesystem etc. 41 /// 42 /// Callers may wish to provide a custom Env object to get fine grain 43 /// control. 44 /// 45 /// All Env implementations are safe for concurrent access from 46 /// multiple threads without any external synchronization. 47 class Env { 48 public: 49 Env(); 50 virtual ~Env() = default; 51 52 /// \brief Returns a default environment suitable for the current operating 53 /// system. 54 /// 55 /// Sophisticated users may wish to provide their own Env 56 /// implementation instead of relying on this default environment. 57 /// 58 /// The result of Default() belongs to this library and must never be deleted. 59 static Env* Default(); 60 61 /// \brief Returns the FileSystem object to handle operations on the file 62 /// specified by 'fname'. The FileSystem object is used as the implementation 63 /// for the file system related (non-virtual) functions that follow. 64 /// Returned FileSystem object is still owned by the Env object and will 65 // (might) be destroyed when the environment is destroyed. 66 virtual Status GetFileSystemForFile(const string& fname, FileSystem** result); 67 68 /// \brief Returns the file system schemes registered for this Env. 69 virtual Status GetRegisteredFileSystemSchemes(std::vector<string>* schemes); 70 71 /// \brief Register a file system for a scheme. 72 virtual Status RegisterFileSystem(const string& scheme, 73 FileSystemRegistry::Factory factory); 74 75 /// \brief Flush filesystem caches for all registered filesystems. 76 Status FlushFileSystemCaches(); 77 78 /// \brief Creates a brand new random access read-only file with the 79 /// specified name. 80 81 /// On success, stores a pointer to the new file in 82 /// *result and returns OK. On failure stores NULL in *result and 83 /// returns non-OK. If the file does not exist, returns a non-OK 84 /// status. 85 /// 86 /// The returned file may be concurrently accessed by multiple threads. 87 /// 88 /// The ownership of the returned RandomAccessFile is passed to the caller 89 /// and the object should be deleted when is not used. The file object 90 /// shouldn't live longer than the Env object. 91 Status NewRandomAccessFile(const string& fname, 92 std::unique_ptr<RandomAccessFile>* result); 93 94 /// \brief Creates an object that writes to a new file with the specified 95 /// name. 96 /// 97 /// Deletes any existing file with the same name and creates a 98 /// new file. On success, stores a pointer to the new file in 99 /// *result and returns OK. On failure stores NULL in *result and 100 /// returns non-OK. 101 /// 102 /// The returned file will only be accessed by one thread at a time. 103 /// 104 /// The ownership of the returned WritableFile is passed to the caller 105 /// and the object should be deleted when is not used. The file object 106 /// shouldn't live longer than the Env object. 107 Status NewWritableFile(const string& fname, 108 std::unique_ptr<WritableFile>* result); 109 110 /// \brief Creates an object that either appends to an existing file, or 111 /// writes to a new file (if the file does not exist to begin with). 112 /// 113 /// On success, stores a pointer to the new file in *result and 114 /// returns OK. On failure stores NULL in *result and returns 115 /// non-OK. 116 /// 117 /// The returned file will only be accessed by one thread at a time. 118 /// 119 /// The ownership of the returned WritableFile is passed to the caller 120 /// and the object should be deleted when is not used. The file object 121 /// shouldn't live longer than the Env object. 122 Status NewAppendableFile(const string& fname, 123 std::unique_ptr<WritableFile>* result); 124 125 /// \brief Creates a readonly region of memory with the file context. 126 /// 127 /// On success, it returns a pointer to read-only memory region 128 /// from the content of file fname. The ownership of the region is passed to 129 /// the caller. On failure stores nullptr in *result and returns non-OK. 130 /// 131 /// The returned memory region can be accessed from many threads in parallel. 132 /// 133 /// The ownership of the returned ReadOnlyMemoryRegion is passed to the caller 134 /// and the object should be deleted when is not used. The memory region 135 /// object shouldn't live longer than the Env object. 136 Status NewReadOnlyMemoryRegionFromFile( 137 const string& fname, std::unique_ptr<ReadOnlyMemoryRegion>* result); 138 139 /// Returns OK if the named path exists and NOT_FOUND otherwise. 140 Status FileExists(const string& fname); 141 142 /// Returns true if all the listed files exist, false otherwise. 143 /// if status is not null, populate the vector with a detailed status 144 /// for each file. 145 bool FilesExist(const std::vector<string>& files, 146 std::vector<Status>* status); 147 148 /// \brief Stores in *result the names of the children of the specified 149 /// directory. The names are relative to "dir". 150 /// 151 /// Original contents of *results are dropped. 152 Status GetChildren(const string& dir, std::vector<string>* result); 153 154 /// \brief Returns true if the path matches the given pattern. The wildcards 155 /// allowed in pattern are described in FileSystem::GetMatchingPaths. 156 virtual bool MatchPath(const string& path, const string& pattern) = 0; 157 158 /// \brief Given a pattern, stores in *results the set of paths that matches 159 /// that pattern. *results is cleared. 160 /// 161 /// More details about `pattern` in FileSystem::GetMatchingPaths. 162 virtual Status GetMatchingPaths(const string& pattern, 163 std::vector<string>* results); 164 165 /// Deletes the named file. 166 Status DeleteFile(const string& fname); 167 168 /// \brief Deletes the specified directory and all subdirectories and files 169 /// underneath it. undeleted_files and undeleted_dirs stores the number of 170 /// files and directories that weren't deleted (unspecified if the return 171 /// status is not OK). 172 /// REQUIRES: undeleted_files, undeleted_dirs to be not null. 173 /// Typical return codes 174 /// * OK - dirname exists and we were able to delete everything underneath. 175 /// * NOT_FOUND - dirname doesn't exist 176 /// * PERMISSION_DENIED - dirname or some descendant is not writable 177 /// * UNIMPLEMENTED - Some underlying functions (like Delete) are not 178 /// implemented 179 Status DeleteRecursively(const string& dirname, int64* undeleted_files, 180 int64* undeleted_dirs); 181 182 /// \brief Creates the specified directory and all the necessary 183 /// subdirectories. Typical return codes. 184 /// * OK - successfully created the directory and sub directories, even if 185 /// they were already created. 186 /// * PERMISSION_DENIED - dirname or some subdirectory is not writable. 187 Status RecursivelyCreateDir(const string& dirname); 188 189 /// \brief Creates the specified directory. Typical return codes 190 /// * OK - successfully created the directory. 191 /// * ALREADY_EXISTS - directory already exists. 192 /// * PERMISSION_DENIED - dirname is not writable. 193 Status CreateDir(const string& dirname); 194 195 /// Deletes the specified directory. 196 Status DeleteDir(const string& dirname); 197 198 /// Obtains statistics for the given path. 199 Status Stat(const string& fname, FileStatistics* stat); 200 201 /// \brief Returns whether the given path is a directory or not. 202 /// Typical return codes (not guaranteed exhaustive): 203 /// * OK - The path exists and is a directory. 204 /// * FAILED_PRECONDITION - The path exists and is not a directory. 205 /// * NOT_FOUND - The path entry does not exist. 206 /// * PERMISSION_DENIED - Insufficient permissions. 207 /// * UNIMPLEMENTED - The file factory doesn't support directories. 208 Status IsDirectory(const string& fname); 209 210 /// Stores the size of `fname` in `*file_size`. 211 Status GetFileSize(const string& fname, uint64* file_size); 212 213 /// \brief Renames file src to target. If target already exists, it will be 214 /// replaced. 215 Status RenameFile(const string& src, const string& target); 216 217 /// \brief Copy the src to target. 218 Status CopyFile(const string& src, const string& target); 219 220 /// \brief Returns the absolute path of the current executable. It resolves 221 /// symlinks if there is any. 222 string GetExecutablePath(); 223 224 /// Creates a local unique temporary file name. Returns true if success. 225 bool LocalTempFilename(string* filename); 226 227 /// Creates a local unique file name that starts with |prefix| and ends with 228 /// |suffix|. Returns true if success. 229 bool CreateUniqueFileName(string* prefix, const string& suffix); 230 231 // TODO(jeff,sanjay): Add back thread/thread-pool support if needed. 232 // TODO(jeff,sanjay): if needed, tighten spec so relative to epoch, or 233 // provide a routine to get the absolute time. 234 235 /// \brief Returns the number of micro-seconds since the Unix epoch. 236 virtual uint64 NowMicros() { return envTime->NowMicros(); }; 237 238 /// \brief Returns the number of seconds since the Unix epoch. 239 virtual uint64 NowSeconds() { return envTime->NowSeconds(); } 240 241 /// Sleeps/delays the thread for the prescribed number of micro-seconds. 242 virtual void SleepForMicroseconds(int64 micros) = 0; 243 244 /// \brief Returns a new thread that is running fn() and is identified 245 /// (for debugging/performance-analysis) by "name". 246 /// 247 /// Caller takes ownership of the result and must delete it eventually 248 /// (the deletion will block until fn() stops running). 249 virtual Thread* StartThread(const ThreadOptions& thread_options, 250 const string& name, 251 std::function<void()> fn) TF_MUST_USE_RESULT = 0; 252 253 // \brief Schedules the given closure on a thread-pool. 254 // 255 // NOTE(mrry): This closure may block. 256 virtual void SchedClosure(std::function<void()> closure) = 0; 257 258 // \brief Schedules the given closure on a thread-pool after the given number 259 // of microseconds. 260 // 261 // NOTE(mrry): This closure must not block. 262 virtual void SchedClosureAfter(int64 micros, 263 std::function<void()> closure) = 0; 264 265 // \brief Load a dynamic library. 266 // 267 // Pass "library_filename" to a platform-specific mechanism for dynamically 268 // loading a library. The rules for determining the exact location of the 269 // library are platform-specific and are not documented here. 270 // 271 // On success, returns a handle to the library in "*handle" and returns 272 // OK from the function. 273 // Otherwise returns nullptr in "*handle" and an error status from the 274 // function. 275 virtual Status LoadLibrary(const char* library_filename, void** handle) = 0; 276 277 // \brief Get a pointer to a symbol from a dynamic library. 278 // 279 // "handle" should be a pointer returned from a previous call to LoadLibrary. 280 // On success, store a pointer to the located symbol in "*symbol" and return 281 // OK from the function. Otherwise, returns nullptr in "*symbol" and an error 282 // status from the function. 283 virtual Status GetSymbolFromLibrary(void* handle, const char* symbol_name, 284 void** symbol) = 0; 285 286 // \brief build the name of dynamic library. 287 // 288 // "name" should be name of the library. 289 // "version" should be the version of the library or NULL 290 // returns the name that LoadLibrary() can use 291 virtual string FormatLibraryFileName(const string& name, 292 const string& version) = 0; 293 294 private: 295 // Returns a possible list of local temporary directories. 296 void GetLocalTempDirectories(std::vector<string>* list); 297 298 std::unique_ptr<FileSystemRegistry> file_system_registry_; 299 TF_DISALLOW_COPY_AND_ASSIGN(Env); 300 EnvTime* envTime = EnvTime::Default(); 301 }; 302 303 /// \brief An implementation of Env that forwards all calls to another Env. 304 /// 305 /// May be useful to clients who wish to override just part of the 306 /// functionality of another Env. 307 class EnvWrapper : public Env { 308 public: 309 /// Initializes an EnvWrapper that delegates all calls to *t 310 explicit EnvWrapper(Env* t) : target_(t) {} 311 virtual ~EnvWrapper(); 312 313 /// Returns the target to which this Env forwards all calls 314 Env* target() const { return target_; } 315 316 Status GetFileSystemForFile(const string& fname, 317 FileSystem** result) override { 318 return target_->GetFileSystemForFile(fname, result); 319 } 320 321 Status GetRegisteredFileSystemSchemes(std::vector<string>* schemes) override { 322 return target_->GetRegisteredFileSystemSchemes(schemes); 323 } 324 325 Status RegisterFileSystem(const string& scheme, 326 FileSystemRegistry::Factory factory) override { 327 return target_->RegisterFileSystem(scheme, factory); 328 } 329 330 bool MatchPath(const string& path, const string& pattern) override { 331 return target_->MatchPath(path, pattern); 332 } 333 334 uint64 NowMicros() override { return target_->NowMicros(); } 335 void SleepForMicroseconds(int64 micros) override { 336 target_->SleepForMicroseconds(micros); 337 } 338 Thread* StartThread(const ThreadOptions& thread_options, const string& name, 339 std::function<void()> fn) override { 340 return target_->StartThread(thread_options, name, fn); 341 } 342 void SchedClosure(std::function<void()> closure) override { 343 target_->SchedClosure(closure); 344 } 345 void SchedClosureAfter(int64 micros, std::function<void()> closure) override { 346 target_->SchedClosureAfter(micros, closure); 347 } 348 Status LoadLibrary(const char* library_filename, void** handle) override { 349 return target_->LoadLibrary(library_filename, handle); 350 } 351 Status GetSymbolFromLibrary(void* handle, const char* symbol_name, 352 void** symbol) override { 353 return target_->GetSymbolFromLibrary(handle, symbol_name, symbol); 354 } 355 string FormatLibraryFileName(const string& name, 356 const string& version) override { 357 return target_->FormatLibraryFileName(name, version); 358 } 359 360 private: 361 Env* target_; 362 }; 363 364 /// Represents a thread used to run a Tensorflow function. 365 class Thread { 366 public: 367 Thread() {} 368 369 /// Blocks until the thread of control stops running. 370 virtual ~Thread(); 371 372 private: 373 TF_DISALLOW_COPY_AND_ASSIGN(Thread); 374 }; 375 376 /// \brief Options to configure a Thread. 377 /// 378 /// Note that the options are all hints, and the 379 /// underlying implementation may choose to ignore it. 380 struct ThreadOptions { 381 /// Thread stack size to use (in bytes). 382 size_t stack_size = 0; // 0: use system default value 383 /// Guard area size to use near thread stacks to use (in bytes) 384 size_t guard_size = 0; // 0: use system default value 385 }; 386 387 /// A utility routine: copy contents of `src` in file system `src_fs` 388 /// to `target` in file system `target_fs`. 389 Status FileSystemCopyFile(FileSystem* src_fs, const string& src, 390 FileSystem* target_fs, const string& target); 391 392 /// A utility routine: reads contents of named file into `*data` 393 Status ReadFileToString(Env* env, const string& fname, string* data); 394 395 /// A utility routine: write contents of `data` to file named `fname` 396 /// (overwriting existing contents, if any). 397 Status WriteStringToFile(Env* env, const string& fname, 398 const StringPiece& data); 399 400 /// Write binary representation of "proto" to the named file. 401 Status WriteBinaryProto(Env* env, const string& fname, 402 const ::tensorflow::protobuf::MessageLite& proto); 403 404 /// Reads contents of named file and parse as binary encoded proto data 405 /// and store into `*proto`. 406 Status ReadBinaryProto(Env* env, const string& fname, 407 ::tensorflow::protobuf::MessageLite* proto); 408 409 /// Write the text representation of "proto" to the named file. 410 Status WriteTextProto(Env* env, const string& fname, 411 const ::tensorflow::protobuf::Message& proto); 412 413 /// Read contents of named file and parse as text encoded proto data 414 /// and store into `*proto`. 415 Status ReadTextProto(Env* env, const string& fname, 416 ::tensorflow::protobuf::Message* proto); 417 418 // START_SKIP_DOXYGEN 419 420 namespace register_file_system { 421 422 template <typename Factory> 423 struct Register { 424 Register(Env* env, const string& scheme) { 425 // TODO(b/32704451): Don't just ignore the ::tensorflow::Status object! 426 env->RegisterFileSystem(scheme, []() -> FileSystem* { return new Factory; }) 427 .IgnoreError(); 428 } 429 }; 430 431 } // namespace register_file_system 432 433 // END_SKIP_DOXYGEN 434 435 } // namespace tensorflow 436 437 // Register a FileSystem implementation for a scheme. Files with names that have 438 // "scheme://" prefixes are routed to use this implementation. 439 #define REGISTER_FILE_SYSTEM_ENV(env, scheme, factory) \ 440 REGISTER_FILE_SYSTEM_UNIQ_HELPER(__COUNTER__, env, scheme, factory) 441 #define REGISTER_FILE_SYSTEM_UNIQ_HELPER(ctr, env, scheme, factory) \ 442 REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory) 443 #define REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory) \ 444 static ::tensorflow::register_file_system::Register<factory> \ 445 register_ff##ctr TF_ATTRIBUTE_UNUSED = \ 446 ::tensorflow::register_file_system::Register<factory>(env, scheme) 447 448 #define REGISTER_FILE_SYSTEM(scheme, factory) \ 449 REGISTER_FILE_SYSTEM_ENV(Env::Default(), scheme, factory); 450 451 #endif // TENSORFLOW_CORE_PLATFORM_ENV_H_ 452