1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #include "tensorflow/core/platform/file_system.h" 17 18 #include <sys/stat.h> 19 20 #include "tensorflow/core/lib/core/status_test_util.h" 21 #include "tensorflow/core/lib/io/path.h" 22 #include "tensorflow/core/lib/strings/str_util.h" 23 #include "tensorflow/core/lib/strings/strcat.h" 24 #include "tensorflow/core/platform/test.h" 25 26 namespace tensorflow { 27 28 static const char* const kPrefix = "ipfs://solarsystem"; 29 30 // A file system that has Planets, Satellites and Sub Satellites. Sub satellites 31 // cannot have children further. 32 class InterPlanetaryFileSystem : public NullFileSystem { 33 public: 34 Status FileExists(const string& fname) override { 35 string parsed_path; 36 ParsePath(fname, &parsed_path); 37 if (BodyExists(parsed_path)) { 38 return Status::OK(); 39 } 40 return Status(tensorflow::error::NOT_FOUND, "File does not exist"); 41 } 42 43 // Adds the dir to the parent's children list and creates an entry for itself. 44 Status CreateDir(const string& dirname) override { 45 string parsed_path; 46 ParsePath(dirname, &parsed_path); 47 // If the directory already exists, throw an error. 48 if (celestial_bodies_.find(parsed_path) != celestial_bodies_.end()) { 49 return Status(tensorflow::error::ALREADY_EXISTS, 50 "dirname already exists."); 51 } 52 std::vector<string> split_path = str_util::Split(parsed_path, '/'); 53 // If the path is too long then we don't support it. 54 if (split_path.size() > 3) { 55 return Status(tensorflow::error::INVALID_ARGUMENT, "Bad dirname"); 56 } 57 if (split_path.empty()) { 58 return Status::OK(); 59 } 60 if (split_path.size() == 1) { 61 celestial_bodies_[""].insert(parsed_path); 62 celestial_bodies_.insert( 63 std::pair<string, std::set<string>>(parsed_path, {})); 64 return Status::OK(); 65 } 66 if (split_path.size() == 2) { 67 if (!BodyExists(split_path[0])) { 68 return Status(tensorflow::error::FAILED_PRECONDITION, 69 "Base dir not created"); 70 } 71 celestial_bodies_[split_path[0]].insert(split_path[1]); 72 celestial_bodies_.insert( 73 std::pair<string, std::set<string>>(parsed_path, {})); 74 return Status::OK(); 75 } 76 if (split_path.size() == 3) { 77 const string& parent_path = io::JoinPath(split_path[0], split_path[1]); 78 if (!BodyExists(parent_path)) { 79 return Status(tensorflow::error::FAILED_PRECONDITION, 80 "Base dir not created"); 81 } 82 celestial_bodies_[parent_path].insert(split_path[2]); 83 celestial_bodies_.insert( 84 std::pair<string, std::set<string>>(parsed_path, {})); 85 return Status::OK(); 86 } 87 return Status(tensorflow::error::FAILED_PRECONDITION, "Failed to create"); 88 } 89 90 Status IsDirectory(const string& dirname) override { 91 string parsed_path; 92 ParsePath(dirname, &parsed_path); 93 // Simulate evil_directory has bad permissions by throwing a LOG(FATAL) 94 if (parsed_path == "evil_directory") { 95 LOG(FATAL) << "evil_directory cannot be accessed"; 96 } 97 std::vector<string> split_path = str_util::Split(parsed_path, '/'); 98 if (split_path.size() > 2) { 99 return Status(tensorflow::error::FAILED_PRECONDITION, "Not a dir"); 100 } 101 if (celestial_bodies_.find(parsed_path) != celestial_bodies_.end()) { 102 return Status::OK(); 103 } 104 return Status(tensorflow::error::FAILED_PRECONDITION, "Not a dir"); 105 } 106 107 Status GetChildren(const string& dir, std::vector<string>* result) override { 108 TF_RETURN_IF_ERROR(IsDirectory(dir)); 109 string parsed_path; 110 ParsePath(dir, &parsed_path); 111 result->insert(result->begin(), celestial_bodies_[parsed_path].begin(), 112 celestial_bodies_[parsed_path].end()); 113 return Status::OK(); 114 } 115 116 private: 117 bool BodyExists(const string& name) { 118 return celestial_bodies_.find(name) != celestial_bodies_.end(); 119 } 120 121 void ParsePath(const string& name, string* parsed_path) { 122 StringPiece scheme, host, path; 123 io::ParseURI(name, &scheme, &host, &path); 124 ASSERT_EQ(scheme, "ipfs"); 125 ASSERT_EQ(host, "solarsystem"); 126 path.Consume("/"); 127 *parsed_path = path.ToString(); 128 } 129 130 std::map<string, std::set<string>> celestial_bodies_ = { 131 std::pair<string, std::set<string>>( 132 "", {"Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", 133 "Uranus", "Neptune"}), 134 std::pair<string, std::set<string>>("Mercury", {}), 135 std::pair<string, std::set<string>>("Venus", {}), 136 std::pair<string, std::set<string>>("Earth", {"Moon"}), 137 std::pair<string, std::set<string>>("Mars", {}), 138 std::pair<string, std::set<string>>("Jupiter", 139 {"Europa", "Io", "Ganymede"}), 140 std::pair<string, std::set<string>>("Saturn", {}), 141 std::pair<string, std::set<string>>("Uranus", {}), 142 std::pair<string, std::set<string>>("Neptune", {}), 143 std::pair<string, std::set<string>>("Earth/Moon", {}), 144 std::pair<string, std::set<string>>("Jupiter/Europa", {}), 145 std::pair<string, std::set<string>>("Jupiter/Io", {}), 146 std::pair<string, std::set<string>>("Jupiter/Ganymede", {})}; 147 }; 148 149 // Returns all the matched entries as a comma separated string removing the 150 // common prefix of BaseDir(). 151 string Match(InterPlanetaryFileSystem* ipfs, const string& suffix_pattern) { 152 std::vector<string> results; 153 Status s = 154 ipfs->GetMatchingPaths(io::JoinPath(kPrefix, suffix_pattern), &results); 155 if (!s.ok()) { 156 return s.ToString(); 157 } else { 158 std::vector<StringPiece> trimmed_results; 159 std::sort(results.begin(), results.end()); 160 for (const string& result : results) { 161 StringPiece trimmed_result(result); 162 EXPECT_TRUE(trimmed_result.Consume(strings::StrCat(kPrefix, "/"))); 163 trimmed_results.push_back(trimmed_result); 164 } 165 return str_util::Join(trimmed_results, ","); 166 } 167 } 168 169 TEST(InterPlanetaryFileSystemTest, IPFSMatch) { 170 InterPlanetaryFileSystem ipfs; 171 EXPECT_EQ(Match(&ipfs, "thereisnosuchfile"), ""); 172 EXPECT_EQ(Match(&ipfs, "*"), 173 "Earth,Jupiter,Mars,Mercury,Neptune,Saturn,Uranus,Venus"); 174 // Returns Jupiter's moons. 175 EXPECT_EQ(Match(&ipfs, "Jupiter/*"), 176 "Jupiter/Europa,Jupiter/Ganymede,Jupiter/Io"); 177 // Returns Jupiter's and Earth's moons. 178 EXPECT_EQ(Match(&ipfs, "*/*"), 179 "Earth/Moon,Jupiter/Europa,Jupiter/Ganymede,Jupiter/Io"); 180 TF_EXPECT_OK(ipfs.CreateDir(io::JoinPath(kPrefix, "Planet0"))); 181 TF_EXPECT_OK(ipfs.CreateDir(io::JoinPath(kPrefix, "Planet1"))); 182 EXPECT_EQ(Match(&ipfs, "Planet[0-1]"), "Planet0,Planet1"); 183 EXPECT_EQ(Match(&ipfs, "Planet?"), "Planet0,Planet1"); 184 } 185 186 TEST(InterPlanetaryFileSystemTest, MatchSimple) { 187 InterPlanetaryFileSystem ipfs; 188 TF_EXPECT_OK(ipfs.CreateDir(io::JoinPath(kPrefix, "match-00"))); 189 TF_EXPECT_OK(ipfs.CreateDir(io::JoinPath(kPrefix, "match-0a"))); 190 TF_EXPECT_OK(ipfs.CreateDir(io::JoinPath(kPrefix, "match-01"))); 191 TF_EXPECT_OK(ipfs.CreateDir(io::JoinPath(kPrefix, "match-aaa"))); 192 193 EXPECT_EQ(Match(&ipfs, "match-*"), "match-00,match-01,match-0a,match-aaa"); 194 EXPECT_EQ(Match(&ipfs, "match-0[0-9]"), "match-00,match-01"); 195 EXPECT_EQ(Match(&ipfs, "match-?[0-9]"), "match-00,match-01"); 196 EXPECT_EQ(Match(&ipfs, "match-?a*"), "match-0a,match-aaa"); 197 EXPECT_EQ(Match(&ipfs, "match-??"), "match-00,match-01,match-0a"); 198 } 199 200 // Create 2 directories abcd and evil_directory. Look for abcd and make sure 201 // that evil_directory isn't accessed. 202 TEST(InterPlanetaryFileSystemTest, MatchOnlyNeeded) { 203 InterPlanetaryFileSystem ipfs; 204 TF_EXPECT_OK(ipfs.CreateDir(io::JoinPath(kPrefix, "abcd"))); 205 TF_EXPECT_OK(ipfs.CreateDir(io::JoinPath(kPrefix, "evil_directory"))); 206 207 EXPECT_EQ(Match(&ipfs, "abcd"), "abcd"); 208 } 209 210 TEST(InterPlanetaryFileSystemTest, MatchDirectory) { 211 InterPlanetaryFileSystem ipfs; 212 TF_EXPECT_OK( 213 ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-00/abc/x"))); 214 TF_EXPECT_OK( 215 ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-0a/abc/x"))); 216 TF_EXPECT_OK( 217 ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-01/abc/x"))); 218 TF_EXPECT_OK( 219 ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-aaa/abc/x"))); 220 221 EXPECT_EQ(Match(&ipfs, "match-*/abc/x"), 222 "match-00/abc/x,match-01/abc/x,match-0a/abc/x,match-aaa/abc/x"); 223 EXPECT_EQ(Match(&ipfs, "match-0[0-9]/abc/x"), 224 "match-00/abc/x,match-01/abc/x"); 225 EXPECT_EQ(Match(&ipfs, "match-?[0-9]/abc/x"), 226 "match-00/abc/x,match-01/abc/x"); 227 EXPECT_EQ(Match(&ipfs, "match-?a*/abc/x"), "match-0a/abc/x,match-aaa/abc/x"); 228 EXPECT_EQ(Match(&ipfs, "match-?[^a]/abc/x"), "match-00/abc/x,match-01/abc/x"); 229 } 230 231 TEST(InterPlanetaryFileSystemTest, MatchMultipleWildcards) { 232 InterPlanetaryFileSystem ipfs; 233 TF_EXPECT_OK( 234 ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-00/abc/00"))); 235 TF_EXPECT_OK( 236 ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-00/abc/01"))); 237 TF_EXPECT_OK( 238 ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-00/abc/09"))); 239 TF_EXPECT_OK( 240 ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-01/abc/00"))); 241 TF_EXPECT_OK( 242 ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-01/abc/04"))); 243 TF_EXPECT_OK( 244 ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-01/abc/10"))); 245 TF_EXPECT_OK( 246 ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-02/abc/00"))); 247 248 EXPECT_EQ(Match(&ipfs, "match-0[0-1]/abc/0[0-8]"), 249 "match-00/abc/00,match-00/abc/01,match-01/abc/00,match-01/abc/04"); 250 } 251 252 TEST(InterPlanetaryFileSystemTest, RecursivelyCreateAlreadyExistingDir) { 253 InterPlanetaryFileSystem ipfs; 254 const string dirname = io::JoinPath(kPrefix, "match-00/abc/00"); 255 TF_EXPECT_OK(ipfs.RecursivelyCreateDir(dirname)); 256 // Ensure that CreateDir throws an error, to sanity check that this test 257 // actually tests the behavior of RecursivelyCreateDir. 258 EXPECT_EQ(ipfs.CreateDir(dirname).code(), tensorflow::error::ALREADY_EXISTS); 259 TF_EXPECT_OK(ipfs.RecursivelyCreateDir(dirname)); 260 } 261 262 // A simple file system with a root directory and a single file underneath it. 263 class TestFileSystem : public NullFileSystem { 264 public: 265 // Only allow for a single root directory. 266 Status IsDirectory(const string& dirname) override { 267 if (dirname == "." || dirname.empty()) { 268 return Status::OK(); 269 } 270 return Status(tensorflow::error::FAILED_PRECONDITION, "Not a dir"); 271 } 272 273 // Simulating a FS with a root dir and a single file underneath it. 274 Status GetChildren(const string& dir, std::vector<string>* result) override { 275 if (dir == "." || dir.empty()) { 276 result->push_back("test"); 277 } 278 return Status::OK(); 279 } 280 }; 281 282 // Making sure that ./<pattern> and <pattern> have the same result. 283 TEST(TestFileSystemTest, RootDirectory) { 284 TestFileSystem fs; 285 std::vector<string> results; 286 auto ret = fs.GetMatchingPaths("./te*", &results); 287 EXPECT_EQ(1, results.size()); 288 EXPECT_EQ("./test", results[0]); 289 ret = fs.GetMatchingPaths("te*", &results); 290 EXPECT_EQ(1, results.size()); 291 EXPECT_EQ("./test", results[0]); 292 } 293 294 } // namespace tensorflow 295
