1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #include <sys/stat.h> 17 #include <algorithm> 18 #include <deque> 19 20 #include "tensorflow/core/lib/core/errors.h" 21 #include "tensorflow/core/lib/core/threadpool.h" 22 #include "tensorflow/core/lib/gtl/map_util.h" 23 #include "tensorflow/core/lib/gtl/stl_util.h" 24 #include "tensorflow/core/lib/io/path.h" 25 #include "tensorflow/core/lib/strings/str_util.h" 26 #include "tensorflow/core/lib/strings/strcat.h" 27 #include "tensorflow/core/platform/env.h" 28 #include "tensorflow/core/platform/file_system.h" 29 #include "tensorflow/core/platform/platform.h" 30 #include "tensorflow/core/platform/protobuf.h" 31 32 namespace tensorflow { 33 34 namespace { 35 36 constexpr int kNumThreads = 8; 37 38 // Run a function in parallel using a ThreadPool, but skip the ThreadPool 39 // on the iOS platform due to its problems with more than a few threads. 40 void ForEach(int first, int last, const std::function<void(int)>& f) { 41 #if TARGET_OS_IPHONE 42 for (int i = first; i < last; i++) { 43 f(i); 44 } 45 #else 46 int num_threads = std::min(kNumThreads, last - first); 47 thread::ThreadPool threads(Env::Default(), "ForEach", num_threads); 48 for (int i = first; i < last; i++) { 49 threads.Schedule([f, i] { f(i); }); 50 } 51 #endif 52 } 53 54 } // anonymous namespace 55 56 FileSystem::~FileSystem() {} 57 58 string FileSystem::TranslateName(const string& name) const { 59 // If the name is empty, CleanPath returns "." which is incorrect and 60 // we should return the empty path instead. 61 if (name.empty()) return name; 62 return io::CleanPath(name); 63 } 64 65 Status FileSystem::IsDirectory(const string& name) { 66 // Check if path exists. 67 TF_RETURN_IF_ERROR(FileExists(name)); 68 FileStatistics stat; 69 TF_RETURN_IF_ERROR(Stat(name, &stat)); 70 if (stat.is_directory) { 71 return Status::OK(); 72 } 73 return Status(tensorflow::error::FAILED_PRECONDITION, "Not a directory"); 74 } 75 76 void FileSystem::FlushCaches() {} 77 78 RandomAccessFile::~RandomAccessFile() {} 79 80 WritableFile::~WritableFile() {} 81 82 FileSystemRegistry::~FileSystemRegistry() {} 83 84 bool FileSystem::FilesExist(const std::vector<string>& files, 85 std::vector<Status>* status) { 86 bool result = true; 87 for (const auto& file : files) { 88 Status s = FileExists(file); 89 result &= s.ok(); 90 if (status != nullptr) { 91 status->push_back(s); 92 } else if (!result) { 93 // Return early since there is no need to check other files. 94 return false; 95 } 96 } 97 return result; 98 } 99 100 Status FileSystem::GetMatchingPaths(const string& pattern, 101 std::vector<string>* results) { 102 results->clear(); 103 // Find the fixed prefix by looking for the first wildcard. 104 string fixed_prefix = pattern.substr(0, pattern.find_first_of("*?[\\")); 105 string eval_pattern = pattern; 106 std::vector<string> all_files; 107 string dir = io::Dirname(fixed_prefix).ToString(); 108 // If dir is empty then we need to fix up fixed_prefix and eval_pattern to 109 // include . as the top level directory. 110 if (dir.empty()) { 111 dir = "."; 112 fixed_prefix = io::JoinPath(dir, fixed_prefix); 113 eval_pattern = io::JoinPath(dir, pattern); 114 } 115 116 // Setup a BFS to explore everything under dir. 117 std::deque<string> dir_q; 118 dir_q.push_back(dir); 119 Status ret; // Status to return. 120 // children_dir_status holds is_dir status for children. It can have three 121 // possible values: OK for true; FAILED_PRECONDITION for false; CANCELLED 122 // if we don't calculate IsDirectory (we might do that because there isn't 123 // any point in exploring that child path). 124 std::vector<Status> children_dir_status; 125 while (!dir_q.empty()) { 126 string current_dir = dir_q.front(); 127 dir_q.pop_front(); 128 std::vector<string> children; 129 Status s = GetChildren(current_dir, &children); 130 ret.Update(s); 131 if (children.empty()) continue; 132 // This IsDirectory call can be expensive for some FS. Parallelizing it. 133 children_dir_status.resize(children.size()); 134 ForEach(0, children.size(), 135 [this, ¤t_dir, &children, &fixed_prefix, 136 &children_dir_status](int i) { 137 const string child_path = io::JoinPath(current_dir, children[i]); 138 // In case the child_path doesn't start with the fixed_prefix then 139 // we don't need to explore this path. 140 if (!StringPiece(child_path).starts_with(fixed_prefix)) { 141 children_dir_status[i] = Status(tensorflow::error::CANCELLED, 142 "Operation not needed"); 143 } else { 144 children_dir_status[i] = IsDirectory(child_path); 145 } 146 }); 147 for (int i = 0; i < children.size(); ++i) { 148 const string child_path = io::JoinPath(current_dir, children[i]); 149 // If the IsDirectory call was cancelled we bail. 150 if (children_dir_status[i].code() == tensorflow::error::CANCELLED) { 151 continue; 152 } 153 // If the child is a directory add it to the queue. 154 if (children_dir_status[i].ok()) { 155 dir_q.push_back(child_path); 156 } 157 all_files.push_back(child_path); 158 } 159 } 160 161 // Match all obtained files to the input pattern. 162 for (const auto& f : all_files) { 163 if (Env::Default()->MatchPath(f, eval_pattern)) { 164 results->push_back(f); 165 } 166 } 167 return ret; 168 } 169 170 Status FileSystem::DeleteRecursively(const string& dirname, 171 int64* undeleted_files, 172 int64* undeleted_dirs) { 173 CHECK_NOTNULL(undeleted_files); 174 CHECK_NOTNULL(undeleted_dirs); 175 176 *undeleted_files = 0; 177 *undeleted_dirs = 0; 178 // Make sure that dirname exists; 179 Status exists_status = FileExists(dirname); 180 if (!exists_status.ok()) { 181 (*undeleted_dirs)++; 182 return exists_status; 183 } 184 std::deque<string> dir_q; // Queue for the BFS 185 std::vector<string> dir_list; // List of all dirs discovered 186 dir_q.push_back(dirname); 187 Status ret; // Status to be returned. 188 // Do a BFS on the directory to discover all the sub-directories. Remove all 189 // children that are files along the way. Then cleanup and remove the 190 // directories in reverse order.; 191 while (!dir_q.empty()) { 192 string dir = dir_q.front(); 193 dir_q.pop_front(); 194 dir_list.push_back(dir); 195 std::vector<string> children; 196 // GetChildren might fail if we don't have appropriate permissions. 197 Status s = GetChildren(dir, &children); 198 ret.Update(s); 199 if (!s.ok()) { 200 (*undeleted_dirs)++; 201 continue; 202 } 203 for (const string& child : children) { 204 const string child_path = io::JoinPath(dir, child); 205 // If the child is a directory add it to the queue, otherwise delete it. 206 if (IsDirectory(child_path).ok()) { 207 dir_q.push_back(child_path); 208 } else { 209 // Delete file might fail because of permissions issues or might be 210 // unimplemented. 211 Status del_status = DeleteFile(child_path); 212 ret.Update(del_status); 213 if (!del_status.ok()) { 214 (*undeleted_files)++; 215 } 216 } 217 } 218 } 219 // Now reverse the list of directories and delete them. The BFS ensures that 220 // we can delete the directories in this order. 221 std::reverse(dir_list.begin(), dir_list.end()); 222 for (const string& dir : dir_list) { 223 // Delete dir might fail because of permissions issues or might be 224 // unimplemented. 225 Status s = DeleteDir(dir); 226 ret.Update(s); 227 if (!s.ok()) { 228 (*undeleted_dirs)++; 229 } 230 } 231 return ret; 232 } 233 234 Status FileSystem::RecursivelyCreateDir(const string& dirname) { 235 StringPiece scheme, host, remaining_dir; 236 io::ParseURI(dirname, &scheme, &host, &remaining_dir); 237 std::vector<StringPiece> sub_dirs; 238 while (!remaining_dir.empty()) { 239 Status status = FileExists(io::CreateURI(scheme, host, remaining_dir)); 240 if (status.ok()) { 241 break; 242 } 243 if (status.code() != error::Code::NOT_FOUND) { 244 return status; 245 } 246 // Basename returns "" for / ending dirs. 247 if (!remaining_dir.ends_with("/")) { 248 sub_dirs.push_back(io::Basename(remaining_dir)); 249 } 250 remaining_dir = io::Dirname(remaining_dir); 251 } 252 253 // sub_dirs contains all the dirs to be created but in reverse order. 254 std::reverse(sub_dirs.begin(), sub_dirs.end()); 255 256 // Now create the directories. 257 string built_path = remaining_dir.ToString(); 258 for (const StringPiece sub_dir : sub_dirs) { 259 built_path = io::JoinPath(built_path, sub_dir); 260 Status status = CreateDir(io::CreateURI(scheme, host, built_path)); 261 if (!status.ok() && status.code() != tensorflow::error::ALREADY_EXISTS) { 262 return status; 263 } 264 } 265 return Status::OK(); 266 } 267 268 Status FileSystem::CopyFile(const string& src, const string& target) { 269 return FileSystemCopyFile(this, src, this, target); 270 } 271 272 } // namespace tensorflow 273