1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #include <dirent.h> 17 #include <errno.h> 18 #include <fcntl.h> 19 #include <stdio.h> 20 #include <sys/mman.h> 21 #if !defined(__APPLE__) 22 #include <sys/sendfile.h> 23 #endif 24 #include <sys/stat.h> 25 #include <sys/time.h> 26 #include <sys/types.h> 27 #include <time.h> 28 #include <unistd.h> 29 30 #include "tensorflow/core/lib/core/error_codes.pb.h" 31 #include "tensorflow/core/lib/core/status.h" 32 #include "tensorflow/core/lib/strings/strcat.h" 33 #include "tensorflow/core/platform/env.h" 34 #include "tensorflow/core/platform/logging.h" 35 #include "tensorflow/core/platform/posix/error.h" 36 #include "tensorflow/core/platform/posix/posix_file_system.h" 37 38 namespace tensorflow { 39 40 // 128KB of copy buffer 41 constexpr size_t kPosixCopyFileBufferSize = 128 * 1024; 42 43 // pread() based random-access 44 class PosixRandomAccessFile : public RandomAccessFile { 45 private: 46 string filename_; 47 int fd_; 48 49 public: 50 PosixRandomAccessFile(const string& fname, int fd) 51 : filename_(fname), fd_(fd) {} 52 ~PosixRandomAccessFile() override { close(fd_); } 53 54 Status Read(uint64 offset, size_t n, StringPiece* result, 55 char* scratch) const override { 56 Status s; 57 char* dst = scratch; 58 while (n > 0 && s.ok()) { 59 ssize_t r = pread(fd_, dst, n, static_cast<off_t>(offset)); 60 if (r > 0) { 61 dst += r; 62 n -= r; 63 offset += r; 64 } else if (r == 0) { 65 s = Status(error::OUT_OF_RANGE, "Read less bytes than requested"); 66 } else if (errno == EINTR || errno == EAGAIN) { 67 // Retry 68 } else { 69 s = IOError(filename_, errno); 70 } 71 } 72 *result = StringPiece(scratch, dst - scratch); 73 return s; 74 } 75 }; 76 77 class PosixWritableFile : public WritableFile { 78 private: 79 string filename_; 80 FILE* file_; 81 82 public: 83 PosixWritableFile(const string& fname, FILE* f) 84 : filename_(fname), file_(f) {} 85 86 ~PosixWritableFile() override { 87 if (file_ != nullptr) { 88 // Ignoring any potential errors 89 fclose(file_); 90 } 91 } 92 93 Status Append(const StringPiece& data) override { 94 size_t r = fwrite(data.data(), 1, data.size(), file_); 95 if (r != data.size()) { 96 return IOError(filename_, errno); 97 } 98 return Status::OK(); 99 } 100 101 Status Close() override { 102 Status result; 103 if (fclose(file_) != 0) { 104 result = IOError(filename_, errno); 105 } 106 file_ = nullptr; 107 return result; 108 } 109 110 Status Flush() override { 111 if (fflush(file_) != 0) { 112 return IOError(filename_, errno); 113 } 114 return Status::OK(); 115 } 116 117 Status Sync() override { 118 Status s; 119 if (fflush(file_) != 0) { 120 s = IOError(filename_, errno); 121 } 122 return s; 123 } 124 }; 125 126 class PosixReadOnlyMemoryRegion : public ReadOnlyMemoryRegion { 127 public: 128 PosixReadOnlyMemoryRegion(const void* address, uint64 length) 129 : address_(address), length_(length) {} 130 ~PosixReadOnlyMemoryRegion() override { 131 munmap(const_cast<void*>(address_), length_); 132 } 133 const void* data() override { return address_; } 134 uint64 length() override { return length_; } 135 136 private: 137 const void* const address_; 138 const uint64 length_; 139 }; 140 141 Status PosixFileSystem::NewRandomAccessFile( 142 const string& fname, std::unique_ptr<RandomAccessFile>* result) { 143 string translated_fname = TranslateName(fname); 144 Status s; 145 int fd = open(translated_fname.c_str(), O_RDONLY); 146 if (fd < 0) { 147 s = IOError(fname, errno); 148 } else { 149 result->reset(new PosixRandomAccessFile(translated_fname, fd)); 150 } 151 return s; 152 } 153 154 Status PosixFileSystem::NewWritableFile(const string& fname, 155 std::unique_ptr<WritableFile>* result) { 156 string translated_fname = TranslateName(fname); 157 Status s; 158 FILE* f = fopen(translated_fname.c_str(), "w"); 159 if (f == nullptr) { 160 s = IOError(fname, errno); 161 } else { 162 result->reset(new PosixWritableFile(translated_fname, f)); 163 } 164 return s; 165 } 166 167 Status PosixFileSystem::NewAppendableFile( 168 const string& fname, std::unique_ptr<WritableFile>* result) { 169 string translated_fname = TranslateName(fname); 170 Status s; 171 FILE* f = fopen(translated_fname.c_str(), "a"); 172 if (f == nullptr) { 173 s = IOError(fname, errno); 174 } else { 175 result->reset(new PosixWritableFile(translated_fname, f)); 176 } 177 return s; 178 } 179 180 Status PosixFileSystem::NewReadOnlyMemoryRegionFromFile( 181 const string& fname, std::unique_ptr<ReadOnlyMemoryRegion>* result) { 182 string translated_fname = TranslateName(fname); 183 Status s = Status::OK(); 184 int fd = open(translated_fname.c_str(), O_RDONLY); 185 if (fd < 0) { 186 s = IOError(fname, errno); 187 } else { 188 struct stat st; 189 ::fstat(fd, &st); 190 const void* address = 191 mmap(nullptr, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); 192 if (address == MAP_FAILED) { 193 s = IOError(fname, errno); 194 } else { 195 result->reset(new PosixReadOnlyMemoryRegion(address, st.st_size)); 196 } 197 close(fd); 198 } 199 return s; 200 } 201 202 Status PosixFileSystem::FileExists(const string& fname) { 203 if (access(TranslateName(fname).c_str(), F_OK) == 0) { 204 return Status::OK(); 205 } 206 return errors::NotFound(fname, " not found"); 207 } 208 209 Status PosixFileSystem::GetChildren(const string& dir, 210 std::vector<string>* result) { 211 string translated_dir = TranslateName(dir); 212 result->clear(); 213 DIR* d = opendir(translated_dir.c_str()); 214 if (d == nullptr) { 215 return IOError(dir, errno); 216 } 217 struct dirent* entry; 218 while ((entry = readdir(d)) != nullptr) { 219 StringPiece basename = entry->d_name; 220 if ((basename != ".") && (basename != "..")) { 221 result->push_back(entry->d_name); 222 } 223 } 224 closedir(d); 225 return Status::OK(); 226 } 227 228 Status PosixFileSystem::DeleteFile(const string& fname) { 229 Status result; 230 if (unlink(TranslateName(fname).c_str()) != 0) { 231 result = IOError(fname, errno); 232 } 233 return result; 234 } 235 236 Status PosixFileSystem::CreateDir(const string& name) { 237 Status result; 238 if (mkdir(TranslateName(name).c_str(), 0755) != 0) { 239 result = IOError(name, errno); 240 } 241 return result; 242 } 243 244 Status PosixFileSystem::DeleteDir(const string& name) { 245 Status result; 246 if (rmdir(TranslateName(name).c_str()) != 0) { 247 result = IOError(name, errno); 248 } 249 return result; 250 } 251 252 Status PosixFileSystem::GetFileSize(const string& fname, uint64* size) { 253 Status s; 254 struct stat sbuf; 255 if (stat(TranslateName(fname).c_str(), &sbuf) != 0) { 256 *size = 0; 257 s = IOError(fname, errno); 258 } else { 259 *size = sbuf.st_size; 260 } 261 return s; 262 } 263 264 Status PosixFileSystem::Stat(const string& fname, FileStatistics* stats) { 265 Status s; 266 struct stat sbuf; 267 if (stat(TranslateName(fname).c_str(), &sbuf) != 0) { 268 s = IOError(fname, errno); 269 } else { 270 stats->length = sbuf.st_size; 271 stats->mtime_nsec = sbuf.st_mtime * 1e9; 272 stats->is_directory = S_ISDIR(sbuf.st_mode); 273 } 274 return s; 275 } 276 277 Status PosixFileSystem::RenameFile(const string& src, const string& target) { 278 Status result; 279 if (rename(TranslateName(src).c_str(), TranslateName(target).c_str()) != 0) { 280 result = IOError(src, errno); 281 } 282 return result; 283 } 284 285 Status PosixFileSystem::CopyFile(const string& src, const string& target) { 286 string translated_src = TranslateName(src); 287 struct stat sbuf; 288 if (stat(translated_src.c_str(), &sbuf) != 0) { 289 return IOError(src, errno); 290 } 291 int src_fd = open(translated_src.c_str(), O_RDONLY); 292 if (src_fd < 0) { 293 return IOError(src, errno); 294 } 295 string translated_target = TranslateName(target); 296 // O_WRONLY | O_CREAT: 297 // Open file for write and if file does not exist, create the file. 298 // S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH: 299 // Create the file with permission of 0644 300 int target_fd = open(translated_target.c_str(), O_WRONLY | O_CREAT, 301 S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); 302 if (target_fd < 0) { 303 close(src_fd); 304 return IOError(target, errno); 305 } 306 int rc = 0; 307 off_t offset = 0; 308 std::unique_ptr<char[]> buffer(new char[kPosixCopyFileBufferSize]); 309 while (offset < sbuf.st_size) { 310 // Use uint64 for safe compare SSIZE_MAX 311 uint64 chunk = sbuf.st_size - offset; 312 if (chunk > SSIZE_MAX) { 313 chunk = SSIZE_MAX; 314 } 315 #if defined(__linux__) && !defined(__ANDROID__) 316 rc = sendfile(target_fd, src_fd, &offset, static_cast<size_t>(chunk)); 317 #else 318 if (chunk > kPosixCopyFileBufferSize) { 319 chunk = kPosixCopyFileBufferSize; 320 } 321 rc = read(src_fd, buffer.get(), static_cast<size_t>(chunk)); 322 if (rc <= 0) { 323 break; 324 } 325 rc = write(target_fd, buffer.get(), static_cast<size_t>(chunk)); 326 offset += chunk; 327 #endif 328 if (rc <= 0) { 329 break; 330 } 331 } 332 333 Status result = Status::OK(); 334 if (rc < 0) { 335 result = IOError(target, errno); 336 } 337 338 // Keep the error code 339 rc = close(target_fd); 340 if (rc < 0 && result == Status::OK()) { 341 result = IOError(target, errno); 342 } 343 rc = close(src_fd); 344 if (rc < 0 && result == Status::OK()) { 345 result = IOError(target, errno); 346 } 347 348 return result; 349 } 350 351 } // namespace tensorflow 352
