1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #include "tensorflow/core/platform/s3/s3_file_system.h" 17 18 #include "tensorflow/core/lib/core/status_test_util.h" 19 #include "tensorflow/core/lib/gtl/stl_util.h" 20 #include "tensorflow/core/lib/io/path.h" 21 #include "tensorflow/core/platform/file_system.h" 22 #include "tensorflow/core/platform/test.h" 23 24 namespace tensorflow { 25 26 namespace { 27 28 class S3FileSystemTest : public ::testing::Test { 29 protected: 30 S3FileSystemTest() {} 31 32 string TmpDir(const string& path) { 33 char* test_dir = getenv("S3_TEST_TMPDIR"); 34 if (test_dir != nullptr) { 35 return io::JoinPath(string(test_dir), path); 36 } else { 37 return "s3://" + io::JoinPath(testing::TmpDir(), path); 38 } 39 } 40 41 Status WriteString(const string& fname, const string& content) { 42 std::unique_ptr<WritableFile> writer; 43 TF_RETURN_IF_ERROR(s3fs.NewWritableFile(fname, &writer)); 44 TF_RETURN_IF_ERROR(writer->Append(content)); 45 TF_RETURN_IF_ERROR(writer->Close()); 46 return Status::OK(); 47 } 48 49 Status ReadAll(const string& fname, string* content) { 50 std::unique_ptr<RandomAccessFile> reader; 51 TF_RETURN_IF_ERROR(s3fs.NewRandomAccessFile(fname, &reader)); 52 53 uint64 file_size = 0; 54 TF_RETURN_IF_ERROR(s3fs.GetFileSize(fname, &file_size)); 55 56 content->resize(file_size); 57 StringPiece result; 58 TF_RETURN_IF_ERROR( 59 reader->Read(0, file_size, &result, gtl::string_as_array(content))); 60 if (file_size != result.size()) { 61 return errors::DataLoss("expected ", file_size, " got ", result.size(), 62 " bytes"); 63 } 64 return Status::OK(); 65 } 66 67 S3FileSystem s3fs; 68 }; 69 70 TEST_F(S3FileSystemTest, NewRandomAccessFile) { 71 const string fname = TmpDir("RandomAccessFile"); 72 const string content = "abcdefghijklmn"; 73 74 TF_ASSERT_OK(WriteString(fname, content)); 75 76 std::unique_ptr<RandomAccessFile> reader; 77 TF_EXPECT_OK(s3fs.NewRandomAccessFile(fname, &reader)); 78 79 string got; 80 got.resize(content.size()); 81 StringPiece result; 82 TF_EXPECT_OK( 83 reader->Read(0, content.size(), &result, gtl::string_as_array(&got))); 84 EXPECT_EQ(content.size(), result.size()); 85 EXPECT_EQ(content, result); 86 87 got.clear(); 88 got.resize(4); 89 TF_EXPECT_OK(reader->Read(2, 4, &result, gtl::string_as_array(&got))); 90 EXPECT_EQ(4, result.size()); 91 EXPECT_EQ(content.substr(2, 4), result); 92 } 93 94 TEST_F(S3FileSystemTest, NewWritableFile) { 95 std::unique_ptr<WritableFile> writer; 96 const string fname = TmpDir("WritableFile"); 97 TF_EXPECT_OK(s3fs.NewWritableFile(fname, &writer)); 98 TF_EXPECT_OK(writer->Append("content1,")); 99 TF_EXPECT_OK(writer->Append("content2")); 100 TF_EXPECT_OK(writer->Flush()); 101 TF_EXPECT_OK(writer->Sync()); 102 TF_EXPECT_OK(writer->Close()); 103 104 string content; 105 TF_EXPECT_OK(ReadAll(fname, &content)); 106 EXPECT_EQ("content1,content2", content); 107 } 108 109 TEST_F(S3FileSystemTest, NewAppendableFile) { 110 std::unique_ptr<WritableFile> writer; 111 112 const string fname = TmpDir("AppendableFile"); 113 TF_ASSERT_OK(WriteString(fname, "test")); 114 115 TF_EXPECT_OK(s3fs.NewAppendableFile(fname, &writer)); 116 TF_EXPECT_OK(writer->Append("content")); 117 TF_EXPECT_OK(writer->Close()); 118 } 119 120 TEST_F(S3FileSystemTest, NewReadOnlyMemoryRegionFromFile) { 121 const string fname = TmpDir("MemoryFile"); 122 const string content = "content"; 123 TF_ASSERT_OK(WriteString(fname, content)); 124 std::unique_ptr<ReadOnlyMemoryRegion> region; 125 TF_EXPECT_OK(s3fs.NewReadOnlyMemoryRegionFromFile(fname, ®ion)); 126 127 EXPECT_EQ(content, StringPiece(reinterpret_cast<const char*>(region->data()), 128 region->length())); 129 } 130 131 TEST_F(S3FileSystemTest, FileExists) { 132 const string fname = TmpDir("FileExists"); 133 // Ensure the file doesn't yet exist. 134 TF_ASSERT_OK(s3fs.DeleteFile(fname)); 135 EXPECT_EQ(error::Code::NOT_FOUND, s3fs.FileExists(fname).code()); 136 TF_ASSERT_OK(WriteString(fname, "test")); 137 TF_EXPECT_OK(s3fs.FileExists(fname)); 138 } 139 140 TEST_F(S3FileSystemTest, GetChildren) { 141 const string base = TmpDir("GetChildren"); 142 TF_EXPECT_OK(s3fs.CreateDir(base)); 143 144 const string file = io::JoinPath(base, "TestFile.csv"); 145 TF_EXPECT_OK(WriteString(file, "test")); 146 147 const string subdir = io::JoinPath(base, "SubDir"); 148 TF_EXPECT_OK(s3fs.CreateDir(subdir)); 149 // s3 object storage doesn't support empty directory, we create file in the 150 // directory 151 const string subfile = io::JoinPath(subdir, "TestSubFile.csv"); 152 TF_EXPECT_OK(WriteString(subfile, "test")); 153 154 std::vector<string> children; 155 TF_EXPECT_OK(s3fs.GetChildren(base, &children)); 156 std::sort(children.begin(), children.end()); 157 EXPECT_EQ(std::vector<string>({"SubDir", "TestFile.csv"}), children); 158 } 159 160 TEST_F(S3FileSystemTest, DeleteFile) { 161 const string fname = TmpDir("DeleteFile"); 162 TF_ASSERT_OK(WriteString(fname, "test")); 163 TF_EXPECT_OK(s3fs.DeleteFile(fname)); 164 } 165 166 TEST_F(S3FileSystemTest, GetFileSize) { 167 const string fname = TmpDir("GetFileSize"); 168 TF_ASSERT_OK(WriteString(fname, "test")); 169 uint64 file_size = 0; 170 TF_EXPECT_OK(s3fs.GetFileSize(fname, &file_size)); 171 EXPECT_EQ(4, file_size); 172 } 173 174 TEST_F(S3FileSystemTest, CreateDir) { 175 // s3 object storage doesn't support empty directory, we create file in the 176 // directory 177 const string dir = TmpDir("CreateDir"); 178 TF_EXPECT_OK(s3fs.CreateDir(dir)); 179 180 const string file = io::JoinPath(dir, "CreateDirFile.csv"); 181 TF_EXPECT_OK(WriteString(file, "test")); 182 FileStatistics stat; 183 TF_EXPECT_OK(s3fs.Stat(dir, &stat)); 184 EXPECT_TRUE(stat.is_directory); 185 } 186 187 TEST_F(S3FileSystemTest, DeleteDir) { 188 // s3 object storage doesn't support empty directory, we create file in the 189 // directory 190 const string dir = TmpDir("DeleteDir"); 191 const string file = io::JoinPath(dir, "DeleteDirFile.csv"); 192 TF_EXPECT_OK(WriteString(file, "test")); 193 EXPECT_FALSE(s3fs.DeleteDir(dir).ok()); 194 195 TF_EXPECT_OK(s3fs.DeleteFile(file)); 196 TF_EXPECT_OK(s3fs.DeleteDir(dir)); 197 FileStatistics stat; 198 EXPECT_FALSE(s3fs.Stat(dir, &stat).ok()); 199 } 200 201 TEST_F(S3FileSystemTest, RenameFile) { 202 const string fname1 = TmpDir("RenameFile1"); 203 const string fname2 = TmpDir("RenameFile2"); 204 TF_ASSERT_OK(WriteString(fname1, "test")); 205 TF_EXPECT_OK(s3fs.RenameFile(fname1, fname2)); 206 string content; 207 TF_EXPECT_OK(ReadAll(fname2, &content)); 208 EXPECT_EQ("test", content); 209 } 210 211 TEST_F(S3FileSystemTest, RenameFile_Overwrite) { 212 const string fname1 = TmpDir("RenameFile1"); 213 const string fname2 = TmpDir("RenameFile2"); 214 215 TF_ASSERT_OK(WriteString(fname2, "test")); 216 TF_EXPECT_OK(s3fs.FileExists(fname2)); 217 218 TF_ASSERT_OK(WriteString(fname1, "test")); 219 TF_EXPECT_OK(s3fs.RenameFile(fname1, fname2)); 220 string content; 221 TF_EXPECT_OK(ReadAll(fname2, &content)); 222 EXPECT_EQ("test", content); 223 } 224 225 TEST_F(S3FileSystemTest, StatFile) { 226 const string fname = TmpDir("StatFile"); 227 TF_ASSERT_OK(WriteString(fname, "test")); 228 FileStatistics stat; 229 TF_EXPECT_OK(s3fs.Stat(fname, &stat)); 230 EXPECT_EQ(4, stat.length); 231 EXPECT_FALSE(stat.is_directory); 232 } 233 234 } // namespace 235 } // namespace tensorflow 236
