1 /* Copyright 2015 Google Inc. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #include <Shlwapi.h> 17 #include <Windows.h> 18 #include <direct.h> 19 #include <errno.h> 20 #include <fcntl.h> 21 #include <io.h> 22 #undef StrCat 23 #include <stdio.h> 24 #include <sys/stat.h> 25 #include <sys/types.h> 26 #include <time.h> 27 28 #include "tensorflow/core/lib/core/error_codes.pb.h" 29 #include "tensorflow/core/lib/strings/strcat.h" 30 #include "tensorflow/core/platform/env.h" 31 #include "tensorflow/core/platform/file_system_helper.h" 32 #include "tensorflow/core/platform/logging.h" 33 #include "tensorflow/core/platform/posix/error.h" 34 #include "tensorflow/core/platform/windows/error.h" 35 #include "tensorflow/core/platform/windows/wide_char.h" 36 #include "tensorflow/core/platform/windows/windows_file_system.h" 37 38 // TODO(mrry): Prevent this Windows.h #define from leaking out of our headers. 39 #undef DeleteFile 40 41 namespace tensorflow { 42 43 namespace { 44 45 // RAII helpers for HANDLEs 46 const auto CloseHandleFunc = [](HANDLE h) { ::CloseHandle(h); }; 47 typedef std::unique_ptr<void, decltype(CloseHandleFunc)> UniqueCloseHandlePtr; 48 49 inline Status IOErrorFromWindowsError(const string& context, DWORD err) { 50 return IOError( 51 context + string(" : ") + internal::GetWindowsErrorMessage(err), err); 52 } 53 54 // PLEASE NOTE: hfile is expected to be an async handle 55 // (i.e. opened with FILE_FLAG_OVERLAPPED) 56 SSIZE_T pread(HANDLE hfile, char* src, size_t num_bytes, uint64_t offset) { 57 assert(num_bytes <= std::numeric_limits<DWORD>::max()); 58 OVERLAPPED overlapped = {0}; 59 ULARGE_INTEGER offset_union; 60 offset_union.QuadPart = offset; 61 62 overlapped.Offset = offset_union.LowPart; 63 overlapped.OffsetHigh = offset_union.HighPart; 64 overlapped.hEvent = ::CreateEvent(NULL, TRUE, FALSE, NULL); 65 66 if (NULL == overlapped.hEvent) { 67 return -1; 68 } 69 70 SSIZE_T result = 0; 71 72 unsigned long bytes_read = 0; 73 DWORD last_error = ERROR_SUCCESS; 74 75 BOOL read_result = ::ReadFile(hfile, src, static_cast<DWORD>(num_bytes), 76 &bytes_read, &overlapped); 77 if (TRUE == read_result) { 78 result = bytes_read; 79 } else if ((FALSE == read_result) && 80 ((last_error = GetLastError()) != ERROR_IO_PENDING)) { 81 result = (last_error == ERROR_HANDLE_EOF) ? 0 : -1; 82 } else { 83 if (ERROR_IO_PENDING == 84 last_error) { // Otherwise bytes_read already has the result. 85 BOOL overlapped_result = 86 ::GetOverlappedResult(hfile, &overlapped, &bytes_read, TRUE); 87 if (FALSE == overlapped_result) { 88 result = (::GetLastError() == ERROR_HANDLE_EOF) ? 0 : -1; 89 } else { 90 result = bytes_read; 91 } 92 } 93 } 94 95 ::CloseHandle(overlapped.hEvent); 96 97 return result; 98 } 99 100 // read() based random-access 101 class WindowsRandomAccessFile : public RandomAccessFile { 102 private: 103 string filename_; 104 HANDLE hfile_; 105 106 public: 107 WindowsRandomAccessFile(const string& fname, HANDLE hfile) 108 : filename_(fname), hfile_(hfile) {} 109 ~WindowsRandomAccessFile() override { 110 if (hfile_ != NULL && hfile_ != INVALID_HANDLE_VALUE) { 111 ::CloseHandle(hfile_); 112 } 113 } 114 115 Status Name(StringPiece* result) const override { 116 *result = filename_; 117 return Status::OK(); 118 } 119 120 Status Read(uint64 offset, size_t n, StringPiece* result, 121 char* scratch) const override { 122 Status s; 123 char* dst = scratch; 124 while (n > 0 && s.ok()) { 125 SSIZE_T r = pread(hfile_, dst, n, offset); 126 if (r > 0) { 127 offset += r; 128 dst += r; 129 n -= r; 130 } else if (r == 0) { 131 s = Status(error::OUT_OF_RANGE, "Read fewer bytes than requested"); 132 } else if (errno == EINTR || errno == EAGAIN) { 133 // Retry 134 } else { 135 s = IOError(filename_, errno); 136 } 137 } 138 *result = StringPiece(scratch, dst - scratch); 139 return s; 140 } 141 }; 142 143 class WindowsWritableFile : public WritableFile { 144 private: 145 string filename_; 146 HANDLE hfile_; 147 148 public: 149 WindowsWritableFile(const string& fname, HANDLE hFile) 150 : filename_(fname), hfile_(hFile) {} 151 152 ~WindowsWritableFile() override { 153 if (hfile_ != NULL && hfile_ != INVALID_HANDLE_VALUE) { 154 WindowsWritableFile::Close(); 155 } 156 } 157 158 Status Append(StringPiece data) override { 159 DWORD bytes_written = 0; 160 DWORD data_size = static_cast<DWORD>(data.size()); 161 BOOL write_result = 162 ::WriteFile(hfile_, data.data(), data_size, &bytes_written, NULL); 163 if (FALSE == write_result) { 164 return IOErrorFromWindowsError("Failed to WriteFile: " + filename_, 165 ::GetLastError()); 166 } 167 168 assert(size_t(bytes_written) == data.size()); 169 return Status::OK(); 170 } 171 172 Status Close() override { 173 assert(INVALID_HANDLE_VALUE != hfile_); 174 175 Status result = Flush(); 176 if (!result.ok()) { 177 return result; 178 } 179 180 if (FALSE == ::CloseHandle(hfile_)) { 181 return IOErrorFromWindowsError("CloseHandle failed for: " + filename_, 182 ::GetLastError()); 183 } 184 185 hfile_ = INVALID_HANDLE_VALUE; 186 return Status::OK(); 187 } 188 189 Status Flush() override { 190 if (FALSE == ::FlushFileBuffers(hfile_)) { 191 return IOErrorFromWindowsError( 192 "FlushFileBuffers failed for: " + filename_, ::GetLastError()); 193 } 194 return Status::OK(); 195 } 196 197 Status Name(StringPiece* result) const override { 198 *result = filename_; 199 return Status::OK(); 200 } 201 202 Status Sync() override { return Flush(); } 203 }; 204 205 class WinReadOnlyMemoryRegion : public ReadOnlyMemoryRegion { 206 private: 207 const std::string filename_; 208 HANDLE hfile_; 209 HANDLE hmap_; 210 211 const void* const address_; 212 const uint64 length_; 213 214 public: 215 WinReadOnlyMemoryRegion(const std::string& filename, HANDLE hfile, 216 HANDLE hmap, const void* address, uint64 length) 217 : filename_(filename), 218 hfile_(hfile), 219 hmap_(hmap), 220 address_(address), 221 length_(length) {} 222 223 ~WinReadOnlyMemoryRegion() { 224 BOOL ret = ::UnmapViewOfFile(address_); 225 assert(ret); 226 227 ret = ::CloseHandle(hmap_); 228 assert(ret); 229 230 ret = ::CloseHandle(hfile_); 231 assert(ret); 232 } 233 234 const void* data() override { return address_; } 235 uint64 length() override { return length_; } 236 }; 237 238 } // namespace 239 240 Status WindowsFileSystem::NewRandomAccessFile( 241 const string& fname, std::unique_ptr<RandomAccessFile>* result) { 242 string translated_fname = TranslateName(fname); 243 std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname); 244 result->reset(); 245 246 // Open the file for read-only random access 247 // Open in async mode which makes Windows allow more parallelism even 248 // if we need to do sync I/O on top of it. 249 DWORD file_flags = FILE_ATTRIBUTE_READONLY | FILE_FLAG_OVERLAPPED; 250 // Shared access is necessary for tests to pass 251 // almost all tests would work with a possible exception of fault_injection. 252 DWORD share_mode = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; 253 254 HANDLE hfile = 255 ::CreateFileW(ws_translated_fname.c_str(), GENERIC_READ, share_mode, NULL, 256 OPEN_EXISTING, file_flags, NULL); 257 258 if (INVALID_HANDLE_VALUE == hfile) { 259 string context = "NewRandomAccessFile failed to Create/Open: " + fname; 260 return IOErrorFromWindowsError(context, ::GetLastError()); 261 } 262 263 result->reset(new WindowsRandomAccessFile(translated_fname, hfile)); 264 return Status::OK(); 265 } 266 267 Status WindowsFileSystem::NewWritableFile( 268 const string& fname, std::unique_ptr<WritableFile>* result) { 269 string translated_fname = TranslateName(fname); 270 std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname); 271 result->reset(); 272 273 DWORD share_mode = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; 274 HANDLE hfile = 275 ::CreateFileW(ws_translated_fname.c_str(), GENERIC_WRITE, share_mode, 276 NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); 277 278 if (INVALID_HANDLE_VALUE == hfile) { 279 string context = "Failed to create a NewWriteableFile: " + fname; 280 return IOErrorFromWindowsError(context, ::GetLastError()); 281 } 282 283 result->reset(new WindowsWritableFile(translated_fname, hfile)); 284 return Status::OK(); 285 } 286 287 Status WindowsFileSystem::NewAppendableFile( 288 const string& fname, std::unique_ptr<WritableFile>* result) { 289 string translated_fname = TranslateName(fname); 290 std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname); 291 result->reset(); 292 293 DWORD share_mode = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; 294 HANDLE hfile = 295 ::CreateFileW(ws_translated_fname.c_str(), GENERIC_WRITE, share_mode, 296 NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); 297 298 if (INVALID_HANDLE_VALUE == hfile) { 299 string context = "Failed to create a NewAppendableFile: " + fname; 300 return IOErrorFromWindowsError(context, ::GetLastError()); 301 } 302 303 UniqueCloseHandlePtr file_guard(hfile, CloseHandleFunc); 304 305 DWORD file_ptr = ::SetFilePointer(hfile, NULL, NULL, FILE_END); 306 if (INVALID_SET_FILE_POINTER == file_ptr) { 307 string context = "Failed to create a NewAppendableFile: " + fname; 308 return IOErrorFromWindowsError(context, ::GetLastError()); 309 } 310 311 result->reset(new WindowsWritableFile(translated_fname, hfile)); 312 file_guard.release(); 313 314 return Status::OK(); 315 } 316 317 Status WindowsFileSystem::NewReadOnlyMemoryRegionFromFile( 318 const string& fname, std::unique_ptr<ReadOnlyMemoryRegion>* result) { 319 string translated_fname = TranslateName(fname); 320 std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname); 321 result->reset(); 322 Status s = Status::OK(); 323 324 // Open the file for read-only 325 DWORD file_flags = FILE_ATTRIBUTE_READONLY; 326 327 // Open in async mode which makes Windows allow more parallelism even 328 // if we need to do sync I/O on top of it. 329 file_flags |= FILE_FLAG_OVERLAPPED; 330 331 DWORD share_mode = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; 332 HANDLE hfile = 333 ::CreateFileW(ws_translated_fname.c_str(), GENERIC_READ, share_mode, NULL, 334 OPEN_EXISTING, file_flags, NULL); 335 336 if (INVALID_HANDLE_VALUE == hfile) { 337 return IOErrorFromWindowsError( 338 "NewReadOnlyMemoryRegionFromFile failed to Create/Open: " + fname, 339 ::GetLastError()); 340 } 341 342 UniqueCloseHandlePtr file_guard(hfile, CloseHandleFunc); 343 344 // Use mmap when virtual address-space is plentiful. 345 uint64_t file_size; 346 s = GetFileSize(translated_fname, &file_size); 347 if (s.ok()) { 348 // Will not map empty files 349 if (file_size == 0) { 350 return IOError( 351 "NewReadOnlyMemoryRegionFromFile failed to map empty file: " + fname, 352 EINVAL); 353 } 354 355 HANDLE hmap = ::CreateFileMappingA(hfile, NULL, PAGE_READONLY, 356 0, // Whole file at its present length 357 0, 358 NULL); // Mapping name 359 360 if (!hmap) { 361 string context = 362 "Failed to create file mapping for " 363 "NewReadOnlyMemoryRegionFromFile: " + 364 fname; 365 return IOErrorFromWindowsError(context, ::GetLastError()); 366 } 367 368 UniqueCloseHandlePtr map_guard(hmap, CloseHandleFunc); 369 370 const void* mapped_region = 371 ::MapViewOfFileEx(hmap, FILE_MAP_READ, 372 0, // High DWORD of access start 373 0, // Low DWORD 374 file_size, 375 NULL); // Let the OS choose the mapping 376 377 if (!mapped_region) { 378 string context = 379 "Failed to MapViewOfFile for " 380 "NewReadOnlyMemoryRegionFromFile: " + 381 fname; 382 return IOErrorFromWindowsError(context, ::GetLastError()); 383 } 384 385 result->reset(new WinReadOnlyMemoryRegion(fname, hfile, hmap, mapped_region, 386 file_size)); 387 388 map_guard.release(); 389 file_guard.release(); 390 } 391 392 return s; 393 } 394 395 Status WindowsFileSystem::FileExists(const string& fname) { 396 constexpr int kOk = 0; 397 std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname)); 398 if (_waccess(ws_translated_fname.c_str(), kOk) == 0) { 399 return Status::OK(); 400 } 401 return errors::NotFound(fname, " not found"); 402 } 403 404 Status WindowsFileSystem::GetChildren(const string& dir, 405 std::vector<string>* result) { 406 string translated_dir = TranslateName(dir); 407 std::wstring ws_translated_dir = Utf8ToWideChar(translated_dir); 408 result->clear(); 409 410 std::wstring pattern = ws_translated_dir; 411 if (!pattern.empty() && pattern.back() != '\\' && pattern.back() != '/') { 412 pattern += L"\\*"; 413 } else { 414 pattern += L'*'; 415 } 416 417 WIN32_FIND_DATAW find_data; 418 HANDLE find_handle = ::FindFirstFileW(pattern.c_str(), &find_data); 419 if (find_handle == INVALID_HANDLE_VALUE) { 420 string context = "FindFirstFile failed for: " + translated_dir; 421 return IOErrorFromWindowsError(context, ::GetLastError()); 422 } 423 424 do { 425 string file_name = WideCharToUtf8(find_data.cFileName); 426 const StringPiece basename = file_name; 427 if (basename != "." && basename != "..") { 428 result->push_back(file_name); 429 } 430 } while (::FindNextFileW(find_handle, &find_data)); 431 432 if (!::FindClose(find_handle)) { 433 string context = "FindClose failed for: " + translated_dir; 434 return IOErrorFromWindowsError(context, ::GetLastError()); 435 } 436 437 return Status::OK(); 438 } 439 440 Status WindowsFileSystem::DeleteFile(const string& fname) { 441 Status result; 442 std::wstring file_name = Utf8ToWideChar(fname); 443 if (_wunlink(file_name.c_str()) != 0) { 444 result = IOError("Failed to delete a file: " + fname, errno); 445 } 446 return result; 447 } 448 449 Status WindowsFileSystem::CreateDir(const string& name) { 450 Status result; 451 std::wstring ws_name = Utf8ToWideChar(name); 452 if (ws_name.empty()) { 453 return errors::AlreadyExists(name); 454 } 455 if (_wmkdir(ws_name.c_str()) != 0) { 456 result = IOError("Failed to create a directory: " + name, errno); 457 } 458 return result; 459 } 460 461 Status WindowsFileSystem::DeleteDir(const string& name) { 462 Status result; 463 std::wstring ws_name = Utf8ToWideChar(name); 464 if (_wrmdir(ws_name.c_str()) != 0) { 465 result = IOError("Failed to remove a directory: " + name, errno); 466 } 467 return result; 468 } 469 470 Status WindowsFileSystem::GetFileSize(const string& fname, uint64* size) { 471 string translated_fname = TranslateName(fname); 472 std::wstring ws_translated_dir = Utf8ToWideChar(translated_fname); 473 Status result; 474 WIN32_FILE_ATTRIBUTE_DATA attrs; 475 if (TRUE == ::GetFileAttributesExW(ws_translated_dir.c_str(), 476 GetFileExInfoStandard, &attrs)) { 477 ULARGE_INTEGER file_size; 478 file_size.HighPart = attrs.nFileSizeHigh; 479 file_size.LowPart = attrs.nFileSizeLow; 480 *size = file_size.QuadPart; 481 } else { 482 string context = "Can not get size for: " + fname; 483 result = IOErrorFromWindowsError(context, ::GetLastError()); 484 } 485 return result; 486 } 487 488 Status WindowsFileSystem::RenameFile(const string& src, const string& target) { 489 Status result; 490 // rename() is not capable of replacing the existing file as on Linux 491 // so use OS API directly 492 std::wstring ws_translated_src = Utf8ToWideChar(TranslateName(src)); 493 std::wstring ws_translated_target = Utf8ToWideChar(TranslateName(target)); 494 if (!::MoveFileExW(ws_translated_src.c_str(), ws_translated_target.c_str(), 495 MOVEFILE_REPLACE_EXISTING)) { 496 string context(strings::StrCat("Failed to rename: ", src, " to: ", target)); 497 result = IOErrorFromWindowsError(context, ::GetLastError()); 498 } 499 return result; 500 } 501 502 Status WindowsFileSystem::GetMatchingPaths(const string& pattern, 503 std::vector<string>* results) { 504 // NOTE(mrry): The existing implementation of FileSystem::GetMatchingPaths() 505 // does not handle Windows paths containing backslashes correctly. Since 506 // Windows APIs will accept forward and backslashes equivalently, we 507 // convert the pattern to use forward slashes exclusively. Note that this 508 // is not ideal, since the API expects backslash as an escape character, 509 // but no code appears to rely on this behavior. 510 string converted_pattern(pattern); 511 std::replace(converted_pattern.begin(), converted_pattern.end(), '\\', '/'); 512 TF_RETURN_IF_ERROR(internal::GetMatchingPaths(this, Env::Default(), 513 converted_pattern, results)); 514 for (string& result : *results) { 515 std::replace(result.begin(), result.end(), '/', '\\'); 516 } 517 return Status::OK(); 518 } 519 520 Status WindowsFileSystem::Stat(const string& fname, FileStatistics* stat) { 521 Status result; 522 struct _stat sbuf; 523 std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname)); 524 if (_wstat(ws_translated_fname.c_str(), &sbuf) != 0) { 525 result = IOError(fname, errno); 526 } else { 527 stat->mtime_nsec = sbuf.st_mtime * 1e9; 528 stat->length = sbuf.st_size; 529 stat->is_directory = PathIsDirectoryW(ws_translated_fname.c_str()); 530 } 531 return result; 532 } 533 534 } // namespace tensorflow 535