Home | History | Annotate | Download | only in windows 
      1 /* Copyright 2015 Google Inc. All Rights Reserved.
      2 
      3 Licensed under the Apache License, Version 2.0 (the "License");
      4 you may not use this file except in compliance with the License.
      5 You may obtain a copy of the License at
      6 
      7     http://www.apache.org/licenses/LICENSE-2.0
      8 
      9 Unless required by applicable law or agreed to in writing, software
     10 distributed under the License is distributed on an "AS IS" BASIS,
     11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     12 See the License for the specific language governing permissions and
     13 limitations under the License.
     14 ==============================================================================*/
     15 
     16 #include <Shlwapi.h>
     17 #include <Windows.h>
     18 #include <direct.h>
     19 #include <errno.h>
     20 #include <fcntl.h>
     21 #include <io.h>
     22 #undef StrCat
     23 #include <stdio.h>
     24 #include <sys/stat.h>
     25 #include <sys/types.h>
     26 #include <time.h>
     27 
     28 #include "tensorflow/core/lib/core/error_codes.pb.h"
     29 #include "tensorflow/core/lib/strings/strcat.h"
     30 #include "tensorflow/core/platform/env.h"
     31 #include "tensorflow/core/platform/file_system_helper.h"
     32 #include "tensorflow/core/platform/logging.h"
     33 #include "tensorflow/core/platform/posix/error.h"
     34 #include "tensorflow/core/platform/windows/error.h"
     35 #include "tensorflow/core/platform/windows/wide_char.h"
     36 #include "tensorflow/core/platform/windows/windows_file_system.h"
     37 
     38 // TODO(mrry): Prevent this Windows.h #define from leaking out of our headers.
     39 #undef DeleteFile
     40 
     41 namespace tensorflow {
     42 
     43 namespace {
     44 
     45 // RAII helpers for HANDLEs
     46 const auto CloseHandleFunc = [](HANDLE h) { ::CloseHandle(h); };
     47 typedef std::unique_ptr<void, decltype(CloseHandleFunc)> UniqueCloseHandlePtr;
     48 
     49 inline Status IOErrorFromWindowsError(const string& context, DWORD err) {
     50   return IOError(
     51       context + string(" : ") + internal::GetWindowsErrorMessage(err), err);
     52 }
     53 
     54 // PLEASE NOTE: hfile is expected to be an async handle
     55 // (i.e. opened with FILE_FLAG_OVERLAPPED)
     56 SSIZE_T pread(HANDLE hfile, char* src, size_t num_bytes, uint64_t offset) {
     57   assert(num_bytes <= std::numeric_limits<DWORD>::max());
     58   OVERLAPPED overlapped = {0};
     59   ULARGE_INTEGER offset_union;
     60   offset_union.QuadPart = offset;
     61 
     62   overlapped.Offset = offset_union.LowPart;
     63   overlapped.OffsetHigh = offset_union.HighPart;
     64   overlapped.hEvent = ::CreateEvent(NULL, TRUE, FALSE, NULL);
     65 
     66   if (NULL == overlapped.hEvent) {
     67     return -1;
     68   }
     69 
     70   SSIZE_T result = 0;
     71 
     72   unsigned long bytes_read = 0;
     73   DWORD last_error = ERROR_SUCCESS;
     74 
     75   BOOL read_result = ::ReadFile(hfile, src, static_cast<DWORD>(num_bytes),
     76                                 &bytes_read, &overlapped);
     77   if (TRUE == read_result) {
     78     result = bytes_read;
     79   } else if ((FALSE == read_result) &&
     80              ((last_error = GetLastError()) != ERROR_IO_PENDING)) {
     81     result = (last_error == ERROR_HANDLE_EOF) ? 0 : -1;
     82   } else {
     83     if (ERROR_IO_PENDING ==
     84         last_error) {  // Otherwise bytes_read already has the result.
     85       BOOL overlapped_result =
     86           ::GetOverlappedResult(hfile, &overlapped, &bytes_read, TRUE);
     87       if (FALSE == overlapped_result) {
     88         result = (::GetLastError() == ERROR_HANDLE_EOF) ? 0 : -1;
     89       } else {
     90         result = bytes_read;
     91       }
     92     }
     93   }
     94 
     95   ::CloseHandle(overlapped.hEvent);
     96 
     97   return result;
     98 }
     99 
    100 // read() based random-access
    101 class WindowsRandomAccessFile : public RandomAccessFile {
    102  private:
    103   string filename_;
    104   HANDLE hfile_;
    105 
    106  public:
    107   WindowsRandomAccessFile(const string& fname, HANDLE hfile)
    108       : filename_(fname), hfile_(hfile) {}
    109   ~WindowsRandomAccessFile() override {
    110     if (hfile_ != NULL && hfile_ != INVALID_HANDLE_VALUE) {
    111       ::CloseHandle(hfile_);
    112     }
    113   }
    114 
    115   Status Name(StringPiece* result) const override {
    116     *result = filename_;
    117     return Status::OK();
    118   }
    119 
    120   Status Read(uint64 offset, size_t n, StringPiece* result,
    121               char* scratch) const override {
    122     Status s;
    123     char* dst = scratch;
    124     while (n > 0 && s.ok()) {
    125       SSIZE_T r = pread(hfile_, dst, n, offset);
    126       if (r > 0) {
    127         offset += r;
    128         dst += r;
    129         n -= r;
    130       } else if (r == 0) {
    131         s = Status(error::OUT_OF_RANGE, "Read fewer bytes than requested");
    132       } else if (errno == EINTR || errno == EAGAIN) {
    133         // Retry
    134       } else {
    135         s = IOError(filename_, errno);
    136       }
    137     }
    138     *result = StringPiece(scratch, dst - scratch);
    139     return s;
    140   }
    141 };
    142 
    143 class WindowsWritableFile : public WritableFile {
    144  private:
    145   string filename_;
    146   HANDLE hfile_;
    147 
    148  public:
    149   WindowsWritableFile(const string& fname, HANDLE hFile)
    150       : filename_(fname), hfile_(hFile) {}
    151 
    152   ~WindowsWritableFile() override {
    153     if (hfile_ != NULL && hfile_ != INVALID_HANDLE_VALUE) {
    154       WindowsWritableFile::Close();
    155     }
    156   }
    157 
    158   Status Append(StringPiece data) override {
    159     DWORD bytes_written = 0;
    160     DWORD data_size = static_cast<DWORD>(data.size());
    161     BOOL write_result =
    162         ::WriteFile(hfile_, data.data(), data_size, &bytes_written, NULL);
    163     if (FALSE == write_result) {
    164       return IOErrorFromWindowsError("Failed to WriteFile: " + filename_,
    165                                      ::GetLastError());
    166     }
    167 
    168     assert(size_t(bytes_written) == data.size());
    169     return Status::OK();
    170   }
    171 
    172   Status Close() override {
    173     assert(INVALID_HANDLE_VALUE != hfile_);
    174 
    175     Status result = Flush();
    176     if (!result.ok()) {
    177       return result;
    178     }
    179 
    180     if (FALSE == ::CloseHandle(hfile_)) {
    181       return IOErrorFromWindowsError("CloseHandle failed for: " + filename_,
    182                                      ::GetLastError());
    183     }
    184 
    185     hfile_ = INVALID_HANDLE_VALUE;
    186     return Status::OK();
    187   }
    188 
    189   Status Flush() override {
    190     if (FALSE == ::FlushFileBuffers(hfile_)) {
    191       return IOErrorFromWindowsError(
    192           "FlushFileBuffers failed for: " + filename_, ::GetLastError());
    193     }
    194     return Status::OK();
    195   }
    196 
    197   Status Name(StringPiece* result) const override {
    198     *result = filename_;
    199     return Status::OK();
    200   }
    201 
    202   Status Sync() override { return Flush(); }
    203 };
    204 
    205 class WinReadOnlyMemoryRegion : public ReadOnlyMemoryRegion {
    206  private:
    207   const std::string filename_;
    208   HANDLE hfile_;
    209   HANDLE hmap_;
    210 
    211   const void* const address_;
    212   const uint64 length_;
    213 
    214  public:
    215   WinReadOnlyMemoryRegion(const std::string& filename, HANDLE hfile,
    216                           HANDLE hmap, const void* address, uint64 length)
    217       : filename_(filename),
    218         hfile_(hfile),
    219         hmap_(hmap),
    220         address_(address),
    221         length_(length) {}
    222 
    223   ~WinReadOnlyMemoryRegion() {
    224     BOOL ret = ::UnmapViewOfFile(address_);
    225     assert(ret);
    226 
    227     ret = ::CloseHandle(hmap_);
    228     assert(ret);
    229 
    230     ret = ::CloseHandle(hfile_);
    231     assert(ret);
    232   }
    233 
    234   const void* data() override { return address_; }
    235   uint64 length() override { return length_; }
    236 };
    237 
    238 }  // namespace
    239 
    240 Status WindowsFileSystem::NewRandomAccessFile(
    241     const string& fname, std::unique_ptr<RandomAccessFile>* result) {
    242   string translated_fname = TranslateName(fname);
    243   std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname);
    244   result->reset();
    245 
    246   // Open the file for read-only random access
    247   // Open in async mode which makes Windows allow more parallelism even
    248   // if we need to do sync I/O on top of it.
    249   DWORD file_flags = FILE_ATTRIBUTE_READONLY | FILE_FLAG_OVERLAPPED;
    250   // Shared access is necessary for tests to pass
    251   // almost all tests would work with a possible exception of fault_injection.
    252   DWORD share_mode = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE;
    253 
    254   HANDLE hfile =
    255       ::CreateFileW(ws_translated_fname.c_str(), GENERIC_READ, share_mode, NULL,
    256                     OPEN_EXISTING, file_flags, NULL);
    257 
    258   if (INVALID_HANDLE_VALUE == hfile) {
    259     string context = "NewRandomAccessFile failed to Create/Open: " + fname;
    260     return IOErrorFromWindowsError(context, ::GetLastError());
    261   }
    262 
    263   result->reset(new WindowsRandomAccessFile(translated_fname, hfile));
    264   return Status::OK();
    265 }
    266 
    267 Status WindowsFileSystem::NewWritableFile(
    268     const string& fname, std::unique_ptr<WritableFile>* result) {
    269   string translated_fname = TranslateName(fname);
    270   std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname);
    271   result->reset();
    272 
    273   DWORD share_mode = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE;
    274   HANDLE hfile =
    275       ::CreateFileW(ws_translated_fname.c_str(), GENERIC_WRITE, share_mode,
    276                     NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
    277 
    278   if (INVALID_HANDLE_VALUE == hfile) {
    279     string context = "Failed to create a NewWriteableFile: " + fname;
    280     return IOErrorFromWindowsError(context, ::GetLastError());
    281   }
    282 
    283   result->reset(new WindowsWritableFile(translated_fname, hfile));
    284   return Status::OK();
    285 }
    286 
    287 Status WindowsFileSystem::NewAppendableFile(
    288     const string& fname, std::unique_ptr<WritableFile>* result) {
    289   string translated_fname = TranslateName(fname);
    290   std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname);
    291   result->reset();
    292 
    293   DWORD share_mode = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE;
    294   HANDLE hfile =
    295       ::CreateFileW(ws_translated_fname.c_str(), GENERIC_WRITE, share_mode,
    296                     NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
    297 
    298   if (INVALID_HANDLE_VALUE == hfile) {
    299     string context = "Failed to create a NewAppendableFile: " + fname;
    300     return IOErrorFromWindowsError(context, ::GetLastError());
    301   }
    302 
    303   UniqueCloseHandlePtr file_guard(hfile, CloseHandleFunc);
    304 
    305   DWORD file_ptr = ::SetFilePointer(hfile, NULL, NULL, FILE_END);
    306   if (INVALID_SET_FILE_POINTER == file_ptr) {
    307     string context = "Failed to create a NewAppendableFile: " + fname;
    308     return IOErrorFromWindowsError(context, ::GetLastError());
    309   }
    310 
    311   result->reset(new WindowsWritableFile(translated_fname, hfile));
    312   file_guard.release();
    313 
    314   return Status::OK();
    315 }
    316 
    317 Status WindowsFileSystem::NewReadOnlyMemoryRegionFromFile(
    318     const string& fname, std::unique_ptr<ReadOnlyMemoryRegion>* result) {
    319   string translated_fname = TranslateName(fname);
    320   std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname);
    321   result->reset();
    322   Status s = Status::OK();
    323 
    324   // Open the file for read-only
    325   DWORD file_flags = FILE_ATTRIBUTE_READONLY;
    326 
    327   // Open in async mode which makes Windows allow more parallelism even
    328   // if we need to do sync I/O on top of it.
    329   file_flags |= FILE_FLAG_OVERLAPPED;
    330 
    331   DWORD share_mode = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE;
    332   HANDLE hfile =
    333       ::CreateFileW(ws_translated_fname.c_str(), GENERIC_READ, share_mode, NULL,
    334                     OPEN_EXISTING, file_flags, NULL);
    335 
    336   if (INVALID_HANDLE_VALUE == hfile) {
    337     return IOErrorFromWindowsError(
    338         "NewReadOnlyMemoryRegionFromFile failed to Create/Open: " + fname,
    339         ::GetLastError());
    340   }
    341 
    342   UniqueCloseHandlePtr file_guard(hfile, CloseHandleFunc);
    343 
    344   // Use mmap when virtual address-space is plentiful.
    345   uint64_t file_size;
    346   s = GetFileSize(translated_fname, &file_size);
    347   if (s.ok()) {
    348     // Will not map empty files
    349     if (file_size == 0) {
    350       return IOError(
    351           "NewReadOnlyMemoryRegionFromFile failed to map empty file: " + fname,
    352           EINVAL);
    353     }
    354 
    355     HANDLE hmap = ::CreateFileMappingA(hfile, NULL, PAGE_READONLY,
    356                                        0,  // Whole file at its present length
    357                                        0,
    358                                        NULL);  // Mapping name
    359 
    360     if (!hmap) {
    361       string context =
    362           "Failed to create file mapping for "
    363           "NewReadOnlyMemoryRegionFromFile: " +
    364           fname;
    365       return IOErrorFromWindowsError(context, ::GetLastError());
    366     }
    367 
    368     UniqueCloseHandlePtr map_guard(hmap, CloseHandleFunc);
    369 
    370     const void* mapped_region =
    371         ::MapViewOfFileEx(hmap, FILE_MAP_READ,
    372                           0,  // High DWORD of access start
    373                           0,  // Low DWORD
    374                           file_size,
    375                           NULL);  // Let the OS choose the mapping
    376 
    377     if (!mapped_region) {
    378       string context =
    379           "Failed to MapViewOfFile for "
    380           "NewReadOnlyMemoryRegionFromFile: " +
    381           fname;
    382       return IOErrorFromWindowsError(context, ::GetLastError());
    383     }
    384 
    385     result->reset(new WinReadOnlyMemoryRegion(fname, hfile, hmap, mapped_region,
    386                                               file_size));
    387 
    388     map_guard.release();
    389     file_guard.release();
    390   }
    391 
    392   return s;
    393 }
    394 
    395 Status WindowsFileSystem::FileExists(const string& fname) {
    396   constexpr int kOk = 0;
    397   std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname));
    398   if (_waccess(ws_translated_fname.c_str(), kOk) == 0) {
    399     return Status::OK();
    400   }
    401   return errors::NotFound(fname, " not found");
    402 }
    403 
    404 Status WindowsFileSystem::GetChildren(const string& dir,
    405                                       std::vector<string>* result) {
    406   string translated_dir = TranslateName(dir);
    407   std::wstring ws_translated_dir = Utf8ToWideChar(translated_dir);
    408   result->clear();
    409 
    410   std::wstring pattern = ws_translated_dir;
    411   if (!pattern.empty() && pattern.back() != '\\' && pattern.back() != '/') {
    412     pattern += L"\\*";
    413   } else {
    414     pattern += L'*';
    415   }
    416 
    417   WIN32_FIND_DATAW find_data;
    418   HANDLE find_handle = ::FindFirstFileW(pattern.c_str(), &find_data);
    419   if (find_handle == INVALID_HANDLE_VALUE) {
    420     string context = "FindFirstFile failed for: " + translated_dir;
    421     return IOErrorFromWindowsError(context, ::GetLastError());
    422   }
    423 
    424   do {
    425     string file_name = WideCharToUtf8(find_data.cFileName);
    426     const StringPiece basename = file_name;
    427     if (basename != "." && basename != "..") {
    428       result->push_back(file_name);
    429     }
    430   } while (::FindNextFileW(find_handle, &find_data));
    431 
    432   if (!::FindClose(find_handle)) {
    433     string context = "FindClose failed for: " + translated_dir;
    434     return IOErrorFromWindowsError(context, ::GetLastError());
    435   }
    436 
    437   return Status::OK();
    438 }
    439 
    440 Status WindowsFileSystem::DeleteFile(const string& fname) {
    441   Status result;
    442   std::wstring file_name = Utf8ToWideChar(fname);
    443   if (_wunlink(file_name.c_str()) != 0) {
    444     result = IOError("Failed to delete a file: " + fname, errno);
    445   }
    446   return result;
    447 }
    448 
    449 Status WindowsFileSystem::CreateDir(const string& name) {
    450   Status result;
    451   std::wstring ws_name = Utf8ToWideChar(name);
    452   if (ws_name.empty()) {
    453     return errors::AlreadyExists(name);
    454   }
    455   if (_wmkdir(ws_name.c_str()) != 0) {
    456     result = IOError("Failed to create a directory: " + name, errno);
    457   }
    458   return result;
    459 }
    460 
    461 Status WindowsFileSystem::DeleteDir(const string& name) {
    462   Status result;
    463   std::wstring ws_name = Utf8ToWideChar(name);
    464   if (_wrmdir(ws_name.c_str()) != 0) {
    465     result = IOError("Failed to remove a directory: " + name, errno);
    466   }
    467   return result;
    468 }
    469 
    470 Status WindowsFileSystem::GetFileSize(const string& fname, uint64* size) {
    471   string translated_fname = TranslateName(fname);
    472   std::wstring ws_translated_dir = Utf8ToWideChar(translated_fname);
    473   Status result;
    474   WIN32_FILE_ATTRIBUTE_DATA attrs;
    475   if (TRUE == ::GetFileAttributesExW(ws_translated_dir.c_str(),
    476                                      GetFileExInfoStandard, &attrs)) {
    477     ULARGE_INTEGER file_size;
    478     file_size.HighPart = attrs.nFileSizeHigh;
    479     file_size.LowPart = attrs.nFileSizeLow;
    480     *size = file_size.QuadPart;
    481   } else {
    482     string context = "Can not get size for: " + fname;
    483     result = IOErrorFromWindowsError(context, ::GetLastError());
    484   }
    485   return result;
    486 }
    487 
    488 Status WindowsFileSystem::RenameFile(const string& src, const string& target) {
    489   Status result;
    490   // rename() is not capable of replacing the existing file as on Linux
    491   // so use OS API directly
    492   std::wstring ws_translated_src = Utf8ToWideChar(TranslateName(src));
    493   std::wstring ws_translated_target = Utf8ToWideChar(TranslateName(target));
    494   if (!::MoveFileExW(ws_translated_src.c_str(), ws_translated_target.c_str(),
    495                      MOVEFILE_REPLACE_EXISTING)) {
    496     string context(strings::StrCat("Failed to rename: ", src, " to: ", target));
    497     result = IOErrorFromWindowsError(context, ::GetLastError());
    498   }
    499   return result;
    500 }
    501 
    502 Status WindowsFileSystem::GetMatchingPaths(const string& pattern,
    503                                            std::vector<string>* results) {
    504   // NOTE(mrry): The existing implementation of FileSystem::GetMatchingPaths()
    505   // does not handle Windows paths containing backslashes correctly. Since
    506   // Windows APIs will accept forward and backslashes equivalently, we
    507   // convert the pattern to use forward slashes exclusively. Note that this
    508   // is not ideal, since the API expects backslash as an escape character,
    509   // but no code appears to rely on this behavior.
    510   string converted_pattern(pattern);
    511   std::replace(converted_pattern.begin(), converted_pattern.end(), '\\', '/');
    512   TF_RETURN_IF_ERROR(internal::GetMatchingPaths(this, Env::Default(),
    513                                                 converted_pattern, results));
    514   for (string& result : *results) {
    515     std::replace(result.begin(), result.end(), '/', '\\');
    516   }
    517   return Status::OK();
    518 }
    519 
    520 Status WindowsFileSystem::Stat(const string& fname, FileStatistics* stat) {
    521   Status result;
    522   struct _stat sbuf;
    523   std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname));
    524   if (_wstat(ws_translated_fname.c_str(), &sbuf) != 0) {
    525     result = IOError(fname, errno);
    526   } else {
    527     stat->mtime_nsec = sbuf.st_mtime * 1e9;
    528     stat->length = sbuf.st_size;
    529     stat->is_directory = PathIsDirectoryW(ws_translated_fname.c_str());
    530   }
    531   return result;
    532 }
    533 
    534 }  // namespace tensorflow
    535