1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #include "tensorflow/core/platform/env.h" 17 18 #include <sys/stat.h> 19 20 #include "tensorflow/core/framework/graph.pb.h" 21 #include "tensorflow/core/framework/node_def.pb.h" 22 #include "tensorflow/core/lib/core/status_test_util.h" 23 #include "tensorflow/core/lib/core/stringpiece.h" 24 #include "tensorflow/core/lib/io/path.h" 25 #include "tensorflow/core/lib/strings/str_util.h" 26 #include "tensorflow/core/lib/strings/strcat.h" 27 #include "tensorflow/core/platform/protobuf.h" 28 #include "tensorflow/core/platform/test.h" 29 30 namespace tensorflow { 31 32 namespace { 33 34 string CreateTestFile(Env* env, const string& filename, int length) { 35 string input(length, 0); 36 for (int i = 0; i < length; i++) input[i] = i; 37 TF_CHECK_OK(WriteStringToFile(env, filename, input)); 38 return input; 39 } 40 41 GraphDef CreateTestProto() { 42 GraphDef g; 43 NodeDef* node = g.add_node(); 44 node->set_name("name1"); 45 node->set_op("op1"); 46 node = g.add_node(); 47 node->set_name("name2"); 48 node->set_op("op2"); 49 return g; 50 } 51 52 } // namespace 53 54 string BaseDir() { return io::JoinPath(testing::TmpDir(), "base_dir"); } 55 56 class DefaultEnvTest : public ::testing::Test { 57 protected: 58 void SetUp() override { TF_CHECK_OK(env_->CreateDir(BaseDir())); } 59 60 void TearDown() override { 61 int64 undeleted_files, undeleted_dirs; 62 TF_CHECK_OK( 63 env_->DeleteRecursively(BaseDir(), &undeleted_files, &undeleted_dirs)); 64 } 65 66 Env* env_ = Env::Default(); 67 }; 68 69 TEST_F(DefaultEnvTest, IncompleteReadOutOfRange) { 70 const string filename = io::JoinPath(BaseDir(), "out_of_range"); 71 const string input = CreateTestFile(env_, filename, 2); 72 std::unique_ptr<RandomAccessFile> f; 73 TF_EXPECT_OK(env_->NewRandomAccessFile(filename, &f)); 74 75 // Reading past EOF should give an OUT_OF_RANGE error 76 StringPiece result; 77 char scratch[3]; 78 EXPECT_EQ(error::OUT_OF_RANGE, f->Read(0, 3, &result, scratch).code()); 79 EXPECT_EQ(input, result); 80 81 // Exact read to EOF works. 82 TF_EXPECT_OK(f->Read(0, 2, &result, scratch)); 83 EXPECT_EQ(input, result); 84 } 85 86 TEST_F(DefaultEnvTest, ReadFileToString) { 87 for (const int length : {0, 1, 1212, 2553, 4928, 8196, 9000, (1 << 20) - 1, 88 1 << 20, (1 << 20) + 1}) { 89 const string filename = strings::StrCat(BaseDir(), "/bar/..//file", length); 90 91 // Write a file with the given length 92 const string input = CreateTestFile(env_, filename, length); 93 94 // Read the file back and check equality 95 string output; 96 TF_EXPECT_OK(ReadFileToString(env_, filename, &output)); 97 EXPECT_EQ(length, output.size()); 98 EXPECT_EQ(input, output); 99 100 // Obtain stats. 101 FileStatistics stat; 102 TF_EXPECT_OK(env_->Stat(filename, &stat)); 103 EXPECT_EQ(length, stat.length); 104 EXPECT_FALSE(stat.is_directory); 105 } 106 } 107 108 TEST_F(DefaultEnvTest, ReadWriteBinaryProto) { 109 const GraphDef proto = CreateTestProto(); 110 const string filename = strings::StrCat(BaseDir(), "binary_proto"); 111 112 // Write the binary proto 113 TF_EXPECT_OK(WriteBinaryProto(env_, filename, proto)); 114 115 // Read the binary proto back in and make sure it's the same. 116 GraphDef result; 117 TF_EXPECT_OK(ReadBinaryProto(env_, filename, &result)); 118 EXPECT_EQ(result.DebugString(), proto.DebugString()); 119 } 120 121 TEST_F(DefaultEnvTest, ReadWriteTextProto) { 122 const GraphDef proto = CreateTestProto(); 123 const string filename = strings::StrCat(BaseDir(), "text_proto"); 124 125 // Write the text proto 126 string as_text; 127 EXPECT_TRUE(protobuf::TextFormat::PrintToString(proto, &as_text)); 128 TF_EXPECT_OK(WriteStringToFile(env_, filename, as_text)); 129 130 // Read the text proto back in and make sure it's the same. 131 GraphDef result; 132 TF_EXPECT_OK(ReadTextProto(env_, filename, &result)); 133 EXPECT_EQ(result.DebugString(), proto.DebugString()); 134 } 135 136 TEST_F(DefaultEnvTest, FileToReadonlyMemoryRegion) { 137 for (const int length : {1, 1212, 2553, 4928, 8196, 9000, (1 << 20) - 1, 138 1 << 20, (1 << 20) + 1}) { 139 const string filename = 140 io::JoinPath(BaseDir(), strings::StrCat("file", length)); 141 142 // Write a file with the given length 143 const string input = CreateTestFile(env_, filename, length); 144 145 // Create the region. 146 std::unique_ptr<ReadOnlyMemoryRegion> region; 147 TF_EXPECT_OK(env_->NewReadOnlyMemoryRegionFromFile(filename, ®ion)); 148 ASSERT_NE(region, nullptr); 149 EXPECT_EQ(length, region->length()); 150 EXPECT_EQ(input, string(reinterpret_cast<const char*>(region->data()), 151 region->length())); 152 FileStatistics stat; 153 TF_EXPECT_OK(env_->Stat(filename, &stat)); 154 EXPECT_EQ(length, stat.length); 155 EXPECT_FALSE(stat.is_directory); 156 } 157 } 158 159 TEST_F(DefaultEnvTest, DeleteRecursively) { 160 // Build a directory structure rooted at root_dir. 161 // root_dir -> dirs: child_dir1, child_dir2; files: root_file1, root_file2 162 // child_dir1 -> files: child1_file1 163 // child_dir2 -> empty 164 const string parent_dir = io::JoinPath(BaseDir(), "root_dir"); 165 const string child_dir1 = io::JoinPath(parent_dir, "child_dir1"); 166 const string child_dir2 = io::JoinPath(parent_dir, "child_dir2"); 167 TF_EXPECT_OK(env_->CreateDir(parent_dir)); 168 const string root_file1 = io::JoinPath(parent_dir, "root_file1"); 169 const string root_file2 = io::JoinPath(parent_dir, "root_file2"); 170 const string root_file3 = io::JoinPath(parent_dir, ".root_file3"); 171 CreateTestFile(env_, root_file1, 100); 172 CreateTestFile(env_, root_file2, 100); 173 CreateTestFile(env_, root_file3, 100); 174 TF_EXPECT_OK(env_->CreateDir(child_dir1)); 175 const string child1_file1 = io::JoinPath(child_dir1, "child1_file1"); 176 CreateTestFile(env_, child1_file1, 100); 177 TF_EXPECT_OK(env_->CreateDir(child_dir2)); 178 179 int64 undeleted_files, undeleted_dirs; 180 TF_EXPECT_OK( 181 env_->DeleteRecursively(parent_dir, &undeleted_files, &undeleted_dirs)); 182 EXPECT_EQ(0, undeleted_files); 183 EXPECT_EQ(0, undeleted_dirs); 184 EXPECT_EQ(error::Code::NOT_FOUND, env_->FileExists(root_file1).code()); 185 EXPECT_EQ(error::Code::NOT_FOUND, env_->FileExists(root_file2).code()); 186 EXPECT_EQ(error::Code::NOT_FOUND, env_->FileExists(root_file3).code()); 187 EXPECT_EQ(error::Code::NOT_FOUND, env_->FileExists(child1_file1).code()); 188 } 189 190 TEST_F(DefaultEnvTest, DeleteRecursivelyFail) { 191 // Try to delete a non-existent directory. 192 const string parent_dir = io::JoinPath(BaseDir(), "root_dir"); 193 194 int64 undeleted_files, undeleted_dirs; 195 Status s = 196 env_->DeleteRecursively(parent_dir, &undeleted_files, &undeleted_dirs); 197 EXPECT_EQ(error::Code::NOT_FOUND, s.code()); 198 EXPECT_EQ(0, undeleted_files); 199 EXPECT_EQ(1, undeleted_dirs); 200 } 201 202 TEST_F(DefaultEnvTest, RecursivelyCreateDir) { 203 const string create_path = io::JoinPath(BaseDir(), "a//b/c/d"); 204 TF_CHECK_OK(env_->RecursivelyCreateDir(create_path)); 205 TF_CHECK_OK(env_->RecursivelyCreateDir(create_path)); // repeat creation. 206 TF_EXPECT_OK(env_->FileExists(create_path)); 207 } 208 209 TEST_F(DefaultEnvTest, RecursivelyCreateDirEmpty) { 210 TF_CHECK_OK(env_->RecursivelyCreateDir("")); 211 } 212 213 TEST_F(DefaultEnvTest, RecursivelyCreateDirSubdirsExist) { 214 // First create a/b. 215 const string subdir_path = io::JoinPath(BaseDir(), "a/b"); 216 TF_CHECK_OK(env_->CreateDir(io::JoinPath(BaseDir(), "a"))); 217 TF_CHECK_OK(env_->CreateDir(subdir_path)); 218 TF_EXPECT_OK(env_->FileExists(subdir_path)); 219 220 // Now try to recursively create a/b/c/d/ 221 const string create_path = io::JoinPath(BaseDir(), "a/b/c/d/"); 222 TF_CHECK_OK(env_->RecursivelyCreateDir(create_path)); 223 TF_CHECK_OK(env_->RecursivelyCreateDir(create_path)); // repeat creation. 224 TF_EXPECT_OK(env_->FileExists(create_path)); 225 TF_EXPECT_OK(env_->FileExists(io::JoinPath(BaseDir(), "a/b/c"))); 226 } 227 228 TEST_F(DefaultEnvTest, LocalFileSystem) { 229 // Test filename with file:// syntax. 230 int expected_num_files = 0; 231 std::vector<string> matching_paths; 232 for (const int length : {0, 1, 1212, 2553, 4928, 8196, 9000, (1 << 20) - 1, 233 1 << 20, (1 << 20) + 1}) { 234 string filename = io::JoinPath(BaseDir(), strings::StrCat("len", length)); 235 236 filename = strings::StrCat("file://", filename); 237 238 // Write a file with the given length 239 const string input = CreateTestFile(env_, filename, length); 240 ++expected_num_files; 241 242 // Ensure that GetMatchingPaths works as intended. 243 TF_EXPECT_OK(env_->GetMatchingPaths( 244 // Try it with the "file://" URI scheme. 245 strings::StrCat("file://", io::JoinPath(BaseDir(), "l*")), 246 &matching_paths)); 247 EXPECT_EQ(expected_num_files, matching_paths.size()); 248 TF_EXPECT_OK(env_->GetMatchingPaths( 249 // Try it without any URI scheme. 250 io::JoinPath(BaseDir(), "l*"), &matching_paths)); 251 EXPECT_EQ(expected_num_files, matching_paths.size()); 252 253 // Read the file back and check equality 254 string output; 255 TF_EXPECT_OK(ReadFileToString(env_, filename, &output)); 256 EXPECT_EQ(length, output.size()); 257 EXPECT_EQ(input, output); 258 259 FileStatistics stat; 260 TF_EXPECT_OK(env_->Stat(filename, &stat)); 261 EXPECT_EQ(length, stat.length); 262 EXPECT_FALSE(stat.is_directory); 263 } 264 } 265 266 TEST_F(DefaultEnvTest, SleepForMicroseconds) { 267 const int64 start = env_->NowMicros(); 268 const int64 sleep_time = 1e6 + 5e5; 269 env_->SleepForMicroseconds(sleep_time); 270 const int64 delta = env_->NowMicros() - start; 271 272 // Subtract 200 from the sleep_time for this check because NowMicros can 273 // sometimes give slightly inconsistent values between the start and the 274 // finish (e.g. because the two calls run on different CPUs). 275 EXPECT_GE(delta, sleep_time - 200); 276 } 277 278 class TmpDirFileSystem : public NullFileSystem { 279 public: 280 Status FileExists(const string& dir) override { 281 StringPiece scheme, host, path; 282 io::ParseURI(dir, &scheme, &host, &path); 283 if (path.empty()) return errors::NotFound(dir, " not found"); 284 // The special "flushed" file exists only if the filesystem's caches have 285 // been flushed. 286 if (path == "/flushed") { 287 if (flushed_) { 288 return Status::OK(); 289 } else { 290 return errors::NotFound("FlushCaches() not called yet"); 291 } 292 } 293 return Env::Default()->FileExists(io::JoinPath(BaseDir(), path)); 294 } 295 296 Status CreateDir(const string& dir) override { 297 StringPiece scheme, host, path; 298 io::ParseURI(dir, &scheme, &host, &path); 299 if (scheme != "tmpdirfs") { 300 return errors::FailedPrecondition("scheme must be tmpdirfs"); 301 } 302 if (host != "testhost") { 303 return errors::FailedPrecondition("host must be testhost"); 304 } 305 return Env::Default()->CreateDir(io::JoinPath(BaseDir(), path)); 306 } 307 308 void FlushCaches() override { flushed_ = true; } 309 310 private: 311 bool flushed_ = false; 312 }; 313 314 REGISTER_FILE_SYSTEM("tmpdirfs", TmpDirFileSystem); 315 316 TEST_F(DefaultEnvTest, FlushFileSystemCaches) { 317 Env* env = Env::Default(); 318 const string flushed = "tmpdirfs://testhost/flushed"; 319 EXPECT_EQ(error::Code::NOT_FOUND, env->FileExists(flushed).code()); 320 TF_EXPECT_OK(env->FlushFileSystemCaches()); 321 TF_EXPECT_OK(env->FileExists(flushed)); 322 } 323 324 TEST_F(DefaultEnvTest, RecursivelyCreateDirWithUri) { 325 Env* env = Env::Default(); 326 const string create_path = "tmpdirfs://testhost/a/b/c/d"; 327 EXPECT_EQ(error::Code::NOT_FOUND, env->FileExists(create_path).code()); 328 TF_CHECK_OK(env->RecursivelyCreateDir(create_path)); 329 TF_CHECK_OK(env->RecursivelyCreateDir(create_path)); // repeat creation. 330 TF_EXPECT_OK(env->FileExists(create_path)); 331 } 332 333 TEST_F(DefaultEnvTest, GetExecutablePath) { 334 Env* env = Env::Default(); 335 TF_EXPECT_OK(env->FileExists(env->GetExecutablePath())); 336 } 337 338 TEST_F(DefaultEnvTest, LocalTempFilename) { 339 Env* env = Env::Default(); 340 string filename; 341 EXPECT_TRUE(env->LocalTempFilename(&filename)); 342 EXPECT_FALSE(env->FileExists(filename).ok()); 343 344 // Write something to the temporary file. 345 std::unique_ptr<WritableFile> file_to_write; 346 TF_CHECK_OK(env->NewWritableFile(filename, &file_to_write)); 347 TF_CHECK_OK(file_to_write->Append("Null")); 348 TF_CHECK_OK(file_to_write->Close()); 349 TF_CHECK_OK(env->FileExists(filename)); 350 351 // Read from the temporary file and check content. 352 std::unique_ptr<RandomAccessFile> file_to_read; 353 TF_CHECK_OK(env->NewRandomAccessFile(filename, &file_to_read)); 354 StringPiece content; 355 char scratch[1024]; 356 CHECK_EQ(error::OUT_OF_RANGE, 357 file_to_read->Read(0 /* offset */, 1024 /* n */, &content, scratch) 358 .code()); 359 EXPECT_EQ("Null", content.ToString()); 360 361 // Delete the temporary file. 362 TF_CHECK_OK(env->DeleteFile(filename)); 363 EXPECT_FALSE(env->FileExists(filename).ok()); 364 } 365 366 TEST_F(DefaultEnvTest, CreateUniqueFileName) { 367 Env* env = Env::Default(); 368 369 string prefix = "tempfile-prefix-"; 370 string suffix = ".tmp"; 371 string filename = prefix; 372 373 EXPECT_TRUE(env->CreateUniqueFileName(&filename, suffix)); 374 375 StringPiece str(filename); 376 EXPECT_TRUE(str.starts_with(prefix)); 377 EXPECT_TRUE(str.ends_with(suffix)); 378 } 379 380 } // namespace tensorflow 381
