1 /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #include "tensorflow/core/platform/cloud/gcs_file_system.h" 17 #include <fstream> 18 #include "tensorflow/core/lib/core/status_test_util.h" 19 #include "tensorflow/core/platform/cloud/http_request_fake.h" 20 #include "tensorflow/core/platform/test.h" 21 22 namespace tensorflow { 23 namespace { 24 25 static GcsFileSystem::TimeoutConfig kTestTimeoutConfig(5, 1, 10, 20, 30); 26 27 class FakeAuthProvider : public AuthProvider { 28 public: 29 Status GetToken(string* token) override { 30 *token = "fake_token"; 31 return Status::OK(); 32 } 33 }; 34 35 TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache) { 36 std::vector<HttpRequest*> requests( 37 {new FakeHttpRequest( 38 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n" 39 "Auth Token: fake_token\n" 40 "Range: 0-5\n" 41 "Timeouts: 5 1 20\n", 42 "012345"), 43 new FakeHttpRequest( 44 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n" 45 "Auth Token: fake_token\n" 46 "Range: 6-11\n" 47 "Timeouts: 5 1 20\n", 48 "6789")}); 49 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 50 std::unique_ptr<HttpRequest::Factory>( 51 new FakeHttpRequestFactory(&requests)), 52 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 53 0 /* stat cache max age */, 0 /* stat cache max entries */, 54 0 /* matching paths cache max age */, 55 0 /* matching paths cache max entries */, 56 0 /* initial retry delay */, kTestTimeoutConfig, 57 nullptr /* gcs additional header */); 58 59 std::unique_ptr<RandomAccessFile> file; 60 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); 61 62 char scratch[6]; 63 StringPiece result; 64 65 // Read the first chunk. 66 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch)); 67 EXPECT_EQ("012345", result); 68 69 // Read the second chunk. 70 EXPECT_EQ( 71 errors::Code::OUT_OF_RANGE, 72 file->Read(sizeof(scratch), sizeof(scratch), &result, scratch).code()); 73 EXPECT_EQ("6789", result); 74 } 75 76 TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache_differentN) { 77 std::vector<HttpRequest*> requests( 78 {new FakeHttpRequest( 79 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n" 80 "Auth Token: fake_token\n" 81 "Range: 0-2\n" 82 "Timeouts: 5 1 20\n", 83 "012"), 84 new FakeHttpRequest( 85 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n" 86 "Auth Token: fake_token\n" 87 "Range: 3-12\n" 88 "Timeouts: 5 1 20\n", 89 "3456789")}); 90 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 91 std::unique_ptr<HttpRequest::Factory>( 92 new FakeHttpRequestFactory(&requests)), 93 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 94 0 /* stat cache max age */, 0 /* stat cache max entries */, 95 0 /* matching paths cache max age */, 96 0 /* matching paths cache max entries */, 97 0 /* initial retry delay */, kTestTimeoutConfig, 98 nullptr /* gcs additional header */); 99 100 std::unique_ptr<RandomAccessFile> file; 101 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); 102 103 char small_scratch[3]; 104 StringPiece result; 105 106 // Read the first chunk. 107 TF_EXPECT_OK(file->Read(0, sizeof(small_scratch), &result, small_scratch)); 108 EXPECT_EQ("012", result); 109 110 // Read the second chunk that is larger. Requires allocation of new buffer. 111 char large_scratch[10]; 112 113 EXPECT_EQ(errors::Code::OUT_OF_RANGE, 114 file->Read(sizeof(small_scratch), sizeof(large_scratch), &result, 115 large_scratch) 116 .code()); 117 EXPECT_EQ("3456789", result); 118 } 119 120 TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache) { 121 // Our underlying file in this test is a 15 byte file with contents 122 // "0123456789abcde". 123 std::vector<HttpRequest*> requests( 124 {new FakeHttpRequest( 125 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n" 126 "Auth Token: fake_token\n" 127 "Range: 0-8\n" 128 "Timeouts: 5 1 20\n", 129 "012345678"), 130 new FakeHttpRequest( 131 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n" 132 "Auth Token: fake_token\n" 133 "Range: 9-17\n" 134 "Timeouts: 5 1 20\n", 135 "9abcde"), 136 new FakeHttpRequest( 137 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n" 138 "Auth Token: fake_token\n" 139 "Range: 18-26\n" 140 "Timeouts: 5 1 20\n", 141 "")}); 142 GcsFileSystem fs( 143 std::unique_ptr<AuthProvider>(new FakeAuthProvider), 144 std::unique_ptr<HttpRequest::Factory>( 145 new FakeHttpRequestFactory(&requests)), 146 9 /* block size */, 18 /* max bytes */, 0 /* max staleness */, 147 0 /* stat cache max age */, 0 /* stat cache max entries */, 148 0 /* matching paths cache max age */, 149 0 /* matching paths cache max entries */, 0 /* initial retry delay */, 150 kTestTimeoutConfig, nullptr /* gcs additional header */); 151 152 char scratch[100]; 153 StringPiece result; 154 { 155 // We are instantiating this in an enclosed scope to make sure after the 156 // unique ptr goes out of scope, we can still access result. 157 std::unique_ptr<RandomAccessFile> file; 158 TF_EXPECT_OK( 159 fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); 160 161 // Read the first chunk. The cache will be populated with the first block of 162 // 9 bytes. 163 scratch[5] = 'x'; 164 TF_EXPECT_OK(file->Read(0, 4, &result, scratch)); 165 EXPECT_EQ("0123", result); 166 EXPECT_EQ(scratch[5], 'x'); // Make sure we only copied 4 bytes. 167 168 // The second chunk will be fully loaded from the cache, no requests are 169 // made. 170 TF_EXPECT_OK(file->Read(4, 4, &result, scratch)); 171 EXPECT_EQ("4567", result); 172 173 // The chunk is only partially cached -- the request will be made to fetch 174 // the next block. 9 bytes will be requested, starting at offset 9. 175 TF_EXPECT_OK(file->Read(6, 5, &result, scratch)); 176 EXPECT_EQ("6789a", result); 177 178 // The range can only be partially satisfied, as the second block contains 179 // only 6 bytes for a total of 9 + 6 = 15 bytes in the file. 180 EXPECT_EQ(errors::Code::OUT_OF_RANGE, 181 file->Read(6, 10, &result, scratch).code()); 182 EXPECT_EQ("6789abcde", result); 183 184 // The range cannot be satisfied, and the requested offset is past the end 185 // of the cache. A new request will be made to read 9 bytes starting at 186 // offset 18. This request will return an empty response, and there will not 187 // be another request. 188 EXPECT_EQ(errors::Code::OUT_OF_RANGE, 189 file->Read(20, 10, &result, scratch).code()); 190 EXPECT_TRUE(result.empty()); 191 192 // The beginning of the file should still be in the LRU cache. There should 193 // not be another request. The buffer size is still 15. 194 TF_EXPECT_OK(file->Read(0, 4, &result, scratch)); 195 } 196 197 EXPECT_EQ("0123", result); 198 } 199 200 TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_Flush) { 201 // Our underlying file in this test is a 15 byte file with contents 202 // "0123456789abcde". 203 std::vector<HttpRequest*> requests( 204 {new FakeHttpRequest( 205 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n" 206 "Auth Token: fake_token\n" 207 "Range: 0-8\n" 208 "Timeouts: 5 1 20\n", 209 "012345678"), 210 new FakeHttpRequest( 211 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n" 212 "Auth Token: fake_token\n" 213 "Range: 0-8\n" 214 "Timeouts: 5 1 20\n", 215 "012345678")}); 216 GcsFileSystem fs( 217 std::unique_ptr<AuthProvider>(new FakeAuthProvider), 218 std::unique_ptr<HttpRequest::Factory>( 219 new FakeHttpRequestFactory(&requests)), 220 9 /* block size */, 18 /* max bytes */, 0 /* max staleness */, 221 0 /* stat cache max age */, 0 /* stat cache max entries */, 222 0 /* matching paths cache max age */, 223 0 /* matching paths cache max entries */, 0 /* initial retry delay */, 224 kTestTimeoutConfig, nullptr /* gcs additional header */); 225 226 char scratch[100]; 227 StringPiece result; 228 std::unique_ptr<RandomAccessFile> file; 229 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); 230 // Read the first chunk. The cache will be populated with the first block of 231 // 9 bytes. 232 scratch[5] = 'x'; 233 TF_EXPECT_OK(file->Read(0, 4, &result, scratch)); 234 EXPECT_EQ("0123", result); 235 EXPECT_EQ(scratch[5], 'x'); // Make sure we only copied 4 bytes. 236 // Flush caches and read the second chunk. This will be a cache miss, and 237 // the same block will be fetched again. 238 fs.FlushCaches(); 239 TF_EXPECT_OK(file->Read(4, 4, &result, scratch)); 240 EXPECT_EQ("4567", result); 241 } 242 243 TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_MaxStaleness) { 244 // Our underlying file in this test is a 16 byte file with contents 245 // "0123456789abcdef". 246 std::vector<HttpRequest*> requests( 247 {new FakeHttpRequest("Uri: https://storage.googleapis.com/bucket/object\n" 248 "Auth Token: fake_token\n" 249 "Range: 0-7\n" 250 "Timeouts: 5 1 20\n", 251 "01234567"), 252 new FakeHttpRequest("Uri: https://storage.googleapis.com/bucket/object\n" 253 "Auth Token: fake_token\n" 254 "Range: 8-15\n" 255 "Timeouts: 5 1 20\n", 256 "89abcdef")}); 257 GcsFileSystem fs( 258 std::unique_ptr<AuthProvider>(new FakeAuthProvider), 259 std::unique_ptr<HttpRequest::Factory>( 260 new FakeHttpRequestFactory(&requests)), 261 8 /* block size */, 16 /* max bytes */, 3600 /* max staleness */, 262 0 /* stat cache max age */, 0 /* stat cache max entries */, 263 0 /* matching paths cache max age */, 264 0 /* matching paths cache max entries */, 0 /* initial retry delay */, 265 kTestTimeoutConfig, nullptr /* gcs additional header */); 266 char scratch[100]; 267 StringPiece result; 268 // There should only be two HTTP requests issued to GCS even though we iterate 269 // this loop 10 times. This shows that the underlying FileBlockCache persists 270 // across file close/open boundaries. 271 for (int i = 0; i < 10; i++) { 272 // Create two files. Since these files have the same name name and the max 273 // staleness of the filesystem is > 0, they will share the same blocks. 274 std::unique_ptr<RandomAccessFile> file1; 275 std::unique_ptr<RandomAccessFile> file2; 276 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/object", &file1)); 277 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/object", &file2)); 278 // Reading the first block from file1 should load it once. 279 TF_EXPECT_OK(file1->Read(0, 8, &result, scratch)); 280 EXPECT_EQ("01234567", result); 281 // Reading the first block from file2 should not trigger a request to load 282 // the first block again, because the FileBlockCache shared by file1 and 283 // file2 already has the first block. 284 TF_EXPECT_OK(file2->Read(0, 8, &result, scratch)); 285 EXPECT_EQ("01234567", result); 286 // Reading the second block from file2 should load it once. 287 TF_EXPECT_OK(file2->Read(8, 8, &result, scratch)); 288 EXPECT_EQ("89abcdef", result); 289 // Reading the second block from file1 should not trigger a request to load 290 // the second block again, because the FileBlockCache shared by file1 and 291 // file2 already has the second block. 292 TF_EXPECT_OK(file1->Read(8, 8, &result, scratch)); 293 EXPECT_EQ("89abcdef", result); 294 } 295 } 296 297 TEST(GcsFileSystemTest, NewRandomAccessFile_NoObjectName) { 298 std::vector<HttpRequest*> requests; 299 GcsFileSystem fs( 300 std::unique_ptr<AuthProvider>(new FakeAuthProvider), 301 std::unique_ptr<HttpRequest::Factory>( 302 new FakeHttpRequestFactory(&requests)), 303 0 /* read ahead bytes */, 0 /* max bytes */, 0 /* max staleness */, 304 0 /* stat cache max age */, 0 /* stat cache max entries */, 305 0 /* matching paths cache max age */, 306 0 /* matching paths cache max entries */, 0 /* initial retry delay */, 307 kTestTimeoutConfig, nullptr /* gcs additional header */); 308 309 std::unique_ptr<RandomAccessFile> file; 310 EXPECT_EQ(errors::Code::INVALID_ARGUMENT, 311 fs.NewRandomAccessFile("gs://bucket/", &file).code()); 312 } 313 314 TEST(GcsFileSystemTest, NewWritableFile) { 315 std::vector<HttpRequest*> requests( 316 {new FakeHttpRequest( 317 "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n" 318 "Auth Token: fake_token\n" 319 "Range: 0-7\n" 320 "Timeouts: 5 1 20\n", 321 "01234567"), 322 new FakeHttpRequest( 323 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?" 324 "uploadType=resumable&name=path%2Fwriteable\n" 325 "Auth Token: fake_token\n" 326 "Header X-Upload-Content-Length: 17\n" 327 "Post: yes\n" 328 "Timeouts: 5 1 10\n", 329 "", {{"Location", "https://custom/upload/location"}}), 330 new FakeHttpRequest("Uri: https://custom/upload/location\n" 331 "Auth Token: fake_token\n" 332 "Header Content-Range: bytes 0-16/17\n" 333 "Timeouts: 5 1 30\n" 334 "Put body: content1,content2\n", 335 ""), 336 new FakeHttpRequest( 337 "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n" 338 "Auth Token: fake_token\n" 339 "Range: 0-7\n" 340 "Timeouts: 5 1 20\n", 341 "01234567")}); 342 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 343 std::unique_ptr<HttpRequest::Factory>( 344 new FakeHttpRequestFactory(&requests)), 345 8 /* block size */, 8 /* max bytes */, 0 /* max staleness */, 346 0 /* stat cache max age */, 0 /* stat cache max entries */, 347 0 /* matching paths cache max age */, 348 0 /* matching paths cache max entries */, 349 0 /* initial retry delay */, kTestTimeoutConfig, 350 nullptr /* gcs additional header */); 351 352 // Read from the file first, to fill the block cache. 353 std::unique_ptr<RandomAccessFile> rfile; 354 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/writeable", &rfile)); 355 char scratch[100]; 356 StringPiece result; 357 TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch)); 358 EXPECT_EQ("0123", result); 359 // Open the writable file. 360 std::unique_ptr<WritableFile> wfile; 361 TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable", &wfile)); 362 TF_EXPECT_OK(wfile->Append("content1,")); 363 TF_EXPECT_OK(wfile->Append("content2")); 364 TF_EXPECT_OK(wfile->Flush()); 365 // Re-reading the file should trigger another HTTP request to GCS. 366 TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch)); 367 EXPECT_EQ("0123", result); 368 // The calls to flush, sync, and close below should not cause uploads because 369 // the file is not dirty. 370 TF_EXPECT_OK(wfile->Flush()); 371 TF_EXPECT_OK(wfile->Sync()); 372 TF_EXPECT_OK(wfile->Close()); 373 } 374 375 TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceeds) { 376 std::vector<HttpRequest*> requests( 377 {new FakeHttpRequest( 378 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?" 379 "uploadType=resumable&name=path%2Fwriteable.txt\n" 380 "Auth Token: fake_token\n" 381 "Header X-Upload-Content-Length: 17\n" 382 "Post: yes\n" 383 "Timeouts: 5 1 10\n", 384 "", {{"Location", "https://custom/upload/location"}}), 385 new FakeHttpRequest("Uri: https://custom/upload/location\n" 386 "Auth Token: fake_token\n" 387 "Header Content-Range: bytes 0-16/17\n" 388 "Timeouts: 5 1 30\n" 389 "Put body: content1,content2\n", 390 "", errors::Unavailable("503"), 503), 391 new FakeHttpRequest("Uri: https://custom/upload/location\n" 392 "Auth Token: fake_token\n" 393 "Timeouts: 5 1 10\n" 394 "Header Content-Range: bytes */17\n" 395 "Put: yes\n", 396 "", errors::FailedPrecondition("308"), nullptr, 397 {{"Range", "0-10"}}, 308), 398 new FakeHttpRequest("Uri: https://custom/upload/location\n" 399 "Auth Token: fake_token\n" 400 "Header Content-Range: bytes 11-16/17\n" 401 "Timeouts: 5 1 30\n" 402 "Put body: ntent2\n", 403 "", errors::Unavailable("503"), 503), 404 new FakeHttpRequest("Uri: https://custom/upload/location\n" 405 "Auth Token: fake_token\n" 406 "Timeouts: 5 1 10\n" 407 "Header Content-Range: bytes */17\n" 408 "Put: yes\n", 409 "", errors::FailedPrecondition("308"), nullptr, 410 {{"Range", "bytes=0-12"}}, 308), 411 new FakeHttpRequest("Uri: https://custom/upload/location\n" 412 "Auth Token: fake_token\n" 413 "Header Content-Range: bytes 13-16/17\n" 414 "Timeouts: 5 1 30\n" 415 "Put body: ent2\n", 416 "")}); 417 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 418 std::unique_ptr<HttpRequest::Factory>( 419 new FakeHttpRequestFactory(&requests)), 420 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 421 0 /* stat cache max age */, 0 /* stat cache max entries */, 422 0 /* matching paths cache max age */, 423 0 /* matching paths cache max entries */, 424 0 /* initial retry delay */, kTestTimeoutConfig, 425 nullptr /* gcs additional header */); 426 427 std::unique_ptr<WritableFile> file; 428 TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); 429 430 TF_EXPECT_OK(file->Append("content1,")); 431 TF_EXPECT_OK(file->Append("content2")); 432 TF_EXPECT_OK(file->Close()); 433 } 434 435 TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceedsOnGetStatus) { 436 // This test also verifies that a file's blocks are purged from the cache when 437 // the file is written, even when the write takes the "succeeds on get status" 438 // path. 439 std::vector<HttpRequest*> requests( 440 {new FakeHttpRequest( 441 "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n" 442 "Auth Token: fake_token\n" 443 "Range: 0-7\n" 444 "Timeouts: 5 1 20\n", 445 "01234567"), 446 new FakeHttpRequest( 447 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?" 448 "uploadType=resumable&name=path%2Fwriteable\n" 449 "Auth Token: fake_token\n" 450 "Header X-Upload-Content-Length: 17\n" 451 "Post: yes\n" 452 "Timeouts: 5 1 10\n", 453 "", {{"Location", "https://custom/upload/location"}}), 454 new FakeHttpRequest("Uri: https://custom/upload/location\n" 455 "Auth Token: fake_token\n" 456 "Header Content-Range: bytes 0-16/17\n" 457 "Timeouts: 5 1 30\n" 458 "Put body: content1,content2\n", 459 "", errors::Unavailable("503"), 503), 460 new FakeHttpRequest("Uri: https://custom/upload/location\n" 461 "Auth Token: fake_token\n" 462 "Timeouts: 5 1 10\n" 463 "Header Content-Range: bytes */17\n" 464 "Put: yes\n", 465 "", Status::OK(), nullptr, {}, 201), 466 new FakeHttpRequest( 467 "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n" 468 "Auth Token: fake_token\n" 469 "Range: 0-7\n" 470 "Timeouts: 5 1 20\n", 471 "01234567")}); 472 GcsFileSystem fs( 473 std::unique_ptr<AuthProvider>(new FakeAuthProvider), 474 std::unique_ptr<HttpRequest::Factory>( 475 new FakeHttpRequestFactory(&requests)), 476 8 /* block size */, 8 /* max bytes */, 3600 /* max staleness */, 477 0 /* stat cache max age */, 0 /* stat cache max entries */, 478 0 /* matching paths cache max age */, 479 0 /* matching paths cache max entries */, 0 /* initial retry delay */, 480 kTestTimeoutConfig, nullptr /* gcs additional header */); 481 // Pull the file's first block into the cache. This will trigger the first 482 // HTTP request to GCS. 483 std::unique_ptr<RandomAccessFile> rfile; 484 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/writeable", &rfile)); 485 char scratch[100]; 486 StringPiece result; 487 TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch)); 488 EXPECT_EQ("0123", result); 489 // Now write to the same file. Once the write succeeds, the cached block will 490 // be flushed. 491 std::unique_ptr<WritableFile> wfile; 492 TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable", &wfile)); 493 TF_EXPECT_OK(wfile->Append("content1,")); 494 TF_EXPECT_OK(wfile->Append("content2")); 495 // Appending doesn't invalidate the read cache - only flushing does. This read 496 // will not trigger an HTTP request to GCS. 497 TF_EXPECT_OK(rfile->Read(4, 4, &result, scratch)); 498 EXPECT_EQ("4567", result); 499 // Closing the file triggers HTTP requests to GCS and invalidates the read 500 // cache for the file. 501 TF_EXPECT_OK(wfile->Close()); 502 // Reading the first block of the file goes to GCS again. 503 TF_EXPECT_OK(rfile->Read(0, 8, &result, scratch)); 504 EXPECT_EQ("01234567", result); 505 } 506 507 TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadAllAttemptsFail) { 508 std::vector<HttpRequest*> requests( 509 {new FakeHttpRequest( 510 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?" 511 "uploadType=resumable&name=path%2Fwriteable.txt\n" 512 "Auth Token: fake_token\n" 513 "Header X-Upload-Content-Length: 17\n" 514 "Post: yes\n" 515 "Timeouts: 5 1 10\n", 516 "", {{"Location", "https://custom/upload/location"}}), 517 new FakeHttpRequest("Uri: https://custom/upload/location\n" 518 "Auth Token: fake_token\n" 519 "Header Content-Range: bytes 0-16/17\n" 520 "Timeouts: 5 1 30\n" 521 "Put body: content1,content2\n", 522 "", errors::Unavailable("503"), 503)}); 523 for (int i = 0; i < 10; i++) { 524 requests.emplace_back(new FakeHttpRequest( 525 "Uri: https://custom/upload/location\n" 526 "Auth Token: fake_token\n" 527 "Timeouts: 5 1 10\n" 528 "Header Content-Range: bytes */17\n" 529 "Put: yes\n", 530 "", errors::FailedPrecondition("important HTTP error 308"), nullptr, 531 {{"Range", "0-10"}}, 308)); 532 requests.emplace_back(new FakeHttpRequest( 533 "Uri: https://custom/upload/location\n" 534 "Auth Token: fake_token\n" 535 "Header Content-Range: bytes 11-16/17\n" 536 "Timeouts: 5 1 30\n" 537 "Put body: ntent2\n", 538 "", errors::Unavailable("important HTTP error 503"), 503)); 539 } 540 // These calls will be made in the Close() attempt from the destructor. 541 // Letting the destructor succeed. 542 requests.emplace_back(new FakeHttpRequest( 543 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?" 544 "uploadType=resumable&name=path%2Fwriteable.txt\n" 545 "Auth Token: fake_token\n" 546 "Header X-Upload-Content-Length: 17\n" 547 "Post: yes\n" 548 "Timeouts: 5 1 10\n", 549 "", {{"Location", "https://custom/upload/location"}})); 550 requests.emplace_back( 551 new FakeHttpRequest("Uri: https://custom/upload/location\n" 552 "Auth Token: fake_token\n" 553 "Header Content-Range: bytes 0-16/17\n" 554 "Timeouts: 5 1 30\n" 555 "Put body: content1,content2\n", 556 "")); 557 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 558 std::unique_ptr<HttpRequest::Factory>( 559 new FakeHttpRequestFactory(&requests)), 560 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 561 0 /* stat cache max age */, 0 /* stat cache max entries */, 562 0 /* matching paths cache max age */, 563 0 /* matching paths cache max entries */, 564 2 /* initial retry delay */, kTestTimeoutConfig, 565 nullptr /* gcs additional header */); 566 567 std::unique_ptr<WritableFile> file; 568 TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); 569 570 TF_EXPECT_OK(file->Append("content1,")); 571 TF_EXPECT_OK(file->Append("content2")); 572 const auto& status = file->Close(); 573 EXPECT_EQ(errors::Code::ABORTED, status.code()); 574 EXPECT_TRUE(StringPiece(status.error_message()) 575 .contains("All 10 retry attempts failed. The last failure: " 576 "Unavailable: important HTTP error 503")) 577 << status; 578 } 579 580 TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) { 581 std::vector<HttpRequest*> requests( 582 {new FakeHttpRequest( 583 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?" 584 "uploadType=resumable&name=path%2Fwriteable.txt\n" 585 "Auth Token: fake_token\n" 586 "Header X-Upload-Content-Length: 17\n" 587 "Post: yes\n" 588 "Timeouts: 5 1 10\n", 589 "", {{"Location", "https://custom/upload/location"}}), 590 new FakeHttpRequest("Uri: https://custom/upload/location\n" 591 "Auth Token: fake_token\n" 592 "Header Content-Range: bytes 0-16/17\n" 593 "Timeouts: 5 1 30\n" 594 "Put body: content1,content2\n", 595 "", errors::NotFound("important HTTP error 410"), 596 410), 597 // These calls will be made in the Close() attempt from the destructor. 598 // Letting the destructor succeed. 599 new FakeHttpRequest( 600 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?" 601 "uploadType=resumable&name=path%2Fwriteable.txt\n" 602 "Auth Token: fake_token\n" 603 "Header X-Upload-Content-Length: 17\n" 604 "Post: yes\n" 605 "Timeouts: 5 1 10\n", 606 "", {{"Location", "https://custom/upload/location"}}), 607 new FakeHttpRequest("Uri: https://custom/upload/location\n" 608 "Auth Token: fake_token\n" 609 "Header Content-Range: bytes 0-16/17\n" 610 "Timeouts: 5 1 30\n" 611 "Put body: content1,content2\n", 612 "")}); 613 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 614 std::unique_ptr<HttpRequest::Factory>( 615 new FakeHttpRequestFactory(&requests)), 616 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 617 0 /* stat cache max age */, 0 /* stat cache max entries */, 618 0 /* matching paths cache max age */, 619 0 /* matching paths cache max entries */, 620 0 /* initial retry delay */, kTestTimeoutConfig, 621 nullptr /* gcs additional header */); 622 623 std::unique_ptr<WritableFile> file; 624 TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); 625 626 TF_EXPECT_OK(file->Append("content1,")); 627 TF_EXPECT_OK(file->Append("content2")); 628 const auto& status = file->Close(); 629 EXPECT_EQ(errors::Code::UNAVAILABLE, status.code()); 630 EXPECT_TRUE( 631 StringPiece(status.error_message()) 632 .contains( 633 "Upload to gs://bucket/path/writeable.txt failed, caused by: " 634 "Not found: important HTTP error 410")) 635 << status; 636 EXPECT_TRUE(StringPiece(status.error_message()) 637 .contains("when uploading gs://bucket/path/writeable.txt")) 638 << status; 639 } 640 641 TEST(GcsFileSystemTest, NewWritableFile_NoObjectName) { 642 std::vector<HttpRequest*> requests; 643 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 644 std::unique_ptr<HttpRequest::Factory>( 645 new FakeHttpRequestFactory(&requests)), 646 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 647 0 /* stat cache max age */, 0 /* stat cache max entries */, 648 0 /* matching paths cache max age */, 649 0 /* matching paths cache max entries */, 650 0 /* initial retry delay */, kTestTimeoutConfig, 651 nullptr /* gcs additional header */); 652 653 std::unique_ptr<WritableFile> file; 654 EXPECT_EQ(errors::Code::INVALID_ARGUMENT, 655 fs.NewWritableFile("gs://bucket/", &file).code()); 656 } 657 658 TEST(GcsFileSystemTest, NewAppendableFile) { 659 std::vector<HttpRequest*> requests( 660 {new FakeHttpRequest( 661 "Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n" 662 "Auth Token: fake_token\n" 663 "Range: 0-31\n" 664 "Timeouts: 5 1 20\n", 665 "content1,"), 666 new FakeHttpRequest( 667 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?" 668 "uploadType=resumable&name=path%2Fappendable\n" 669 "Auth Token: fake_token\n" 670 "Header X-Upload-Content-Length: 17\n" 671 "Post: yes\n" 672 "Timeouts: 5 1 10\n", 673 "", {{"Location", "https://custom/upload/location"}}), 674 new FakeHttpRequest("Uri: https://custom/upload/location\n" 675 "Auth Token: fake_token\n" 676 "Header Content-Range: bytes 0-16/17\n" 677 "Timeouts: 5 1 30\n" 678 "Put body: content1,content2\n", 679 ""), 680 new FakeHttpRequest( 681 "Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n" 682 "Auth Token: fake_token\n" 683 "Range: 0-31\n" 684 "Timeouts: 5 1 20\n", 685 "01234567")}); 686 GcsFileSystem fs( 687 std::unique_ptr<AuthProvider>(new FakeAuthProvider), 688 std::unique_ptr<HttpRequest::Factory>( 689 new FakeHttpRequestFactory(&requests)), 690 32 /* block size */, 32 /* max bytes */, 0 /* max staleness */, 691 0 /* stat cache max age */, 0 /* stat cache max entries */, 692 0 /* matching paths cache max age */, 693 0 /* matching paths cache max entries */, 0 /* initial retry delay */, 694 kTestTimeoutConfig, nullptr /* gcs additional header */); 695 696 // Create an appendable file. This should read the file from GCS, and pull its 697 // contents into the block cache. 698 std::unique_ptr<WritableFile> wfile; 699 TF_EXPECT_OK(fs.NewAppendableFile("gs://bucket/path/appendable", &wfile)); 700 TF_EXPECT_OK(wfile->Append("content2")); 701 // Verify that the file contents are in the block cache. This read should not 702 // trigger an HTTP request to GCS. 703 std::unique_ptr<RandomAccessFile> rfile; 704 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/appendable", &rfile)); 705 char scratch[100]; 706 StringPiece result; 707 TF_EXPECT_OK(rfile->Read(0, 8, &result, scratch)); 708 EXPECT_EQ("content1", result); 709 // Closing the appendable file will flush its contents to GCS, triggering HTTP 710 // requests. 711 TF_EXPECT_OK(wfile->Close()); 712 // Redo the read. The block should be reloaded from GCS, causing one more HTTP 713 // request to load it. 714 TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch)); 715 EXPECT_EQ("0123", result); 716 } 717 718 TEST(GcsFileSystemTest, NewAppendableFile_NoObjectName) { 719 std::vector<HttpRequest*> requests; 720 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 721 std::unique_ptr<HttpRequest::Factory>( 722 new FakeHttpRequestFactory(&requests)), 723 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 724 0 /* stat cache max age */, 0 /* stat cache max entries */, 725 0 /* matching paths cache max age */, 726 0 /* matching paths cache max entries */, 727 0 /* initial retry delay */, kTestTimeoutConfig, 728 nullptr /* gcs additional header */); 729 730 std::unique_ptr<WritableFile> file; 731 EXPECT_EQ(errors::Code::INVALID_ARGUMENT, 732 fs.NewAppendableFile("gs://bucket/", &file).code()); 733 } 734 735 TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) { 736 const string content = "file content"; 737 std::vector<HttpRequest*> requests( 738 {new FakeHttpRequest( 739 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 740 "path%2Frandom_access.txt?fields=size%2Cupdated\n" 741 "Auth Token: fake_token\n" 742 "Timeouts: 5 1 10\n", 743 strings::StrCat("{\"size\": \"", content.size(), 744 "\", \"updated\": \"2016-04-29T23:15:24.896Z\"}")), 745 new FakeHttpRequest( 746 strings::StrCat("Uri: https://storage.googleapis.com/bucket/" 747 "path%2Frandom_access.txt\n" 748 "Auth Token: fake_token\n" 749 "Range: 0-", 750 content.size() - 1, "\n", "Timeouts: 5 1 20\n"), 751 content)}); 752 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 753 std::unique_ptr<HttpRequest::Factory>( 754 new FakeHttpRequestFactory(&requests)), 755 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 756 0 /* stat cache max age */, 0 /* stat cache max entries */, 757 0 /* matching paths cache max age */, 758 0 /* matching paths cache max entries */, 759 0 /* initial retry delay */, kTestTimeoutConfig, 760 nullptr /* gcs additional header */); 761 762 std::unique_ptr<ReadOnlyMemoryRegion> region; 763 TF_EXPECT_OK(fs.NewReadOnlyMemoryRegionFromFile( 764 "gs://bucket/path/random_access.txt", ®ion)); 765 766 EXPECT_EQ(content, StringPiece(reinterpret_cast<const char*>(region->data()), 767 region->length())); 768 } 769 770 TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile_NoObjectName) { 771 std::vector<HttpRequest*> requests; 772 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 773 std::unique_ptr<HttpRequest::Factory>( 774 new FakeHttpRequestFactory(&requests)), 775 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 776 0 /* stat cache max age */, 0 /* stat cache max entries */, 777 0 /* matching paths cache max age */, 778 0 /* matching paths cache max entries */, 779 0 /* initial retry delay */, kTestTimeoutConfig, 780 nullptr /* gcs additional header */); 781 782 std::unique_ptr<ReadOnlyMemoryRegion> region; 783 EXPECT_EQ(errors::Code::INVALID_ARGUMENT, 784 fs.NewReadOnlyMemoryRegionFromFile("gs://bucket/", ®ion).code()); 785 } 786 787 TEST(GcsFileSystemTest, FileExists_YesAsObject) { 788 std::vector<HttpRequest*> requests({new FakeHttpRequest( 789 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 790 "path%2Ffile1.txt?fields=size%2Cupdated\n" 791 "Auth Token: fake_token\n" 792 "Timeouts: 5 1 10\n", 793 strings::StrCat("{\"size\": \"1010\"," 794 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); 795 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 796 std::unique_ptr<HttpRequest::Factory>( 797 new FakeHttpRequestFactory(&requests)), 798 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 799 0 /* stat cache max age */, 0 /* stat cache max entries */, 800 0 /* matching paths cache max age */, 801 0 /* matching paths cache max entries */, 802 0 /* initial retry delay */, kTestTimeoutConfig, 803 nullptr /* gcs additional header */); 804 805 TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt")); 806 } 807 808 TEST(GcsFileSystemTest, FileExists_YesAsFolder) { 809 std::vector<HttpRequest*> requests( 810 {new FakeHttpRequest( 811 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 812 "path%2Fsubfolder?fields=size%2Cupdated\n" 813 "Auth Token: fake_token\n" 814 "Timeouts: 5 1 10\n", 815 "", errors::NotFound("404"), 404), 816 new FakeHttpRequest( 817 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 818 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubfolder%2F" 819 "&maxResults=1\n" 820 "Auth Token: fake_token\n" 821 "Timeouts: 5 1 10\n", 822 "{\"items\": [ " 823 " { \"name\": \"path/subfolder/\" }]}")}); 824 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 825 std::unique_ptr<HttpRequest::Factory>( 826 new FakeHttpRequestFactory(&requests)), 827 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 828 0 /* stat cache max age */, 0 /* stat cache max entries */, 829 0 /* matching paths cache max age */, 830 0 /* matching paths cache max entries */, 831 0 /* initial retry delay */, kTestTimeoutConfig, 832 nullptr /* gcs additional header */); 833 834 TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder")); 835 } 836 837 TEST(GcsFileSystemTest, FileExists_YesAsBucket) { 838 std::vector<HttpRequest*> requests( 839 {new FakeHttpRequest( 840 "Uri: https://www.googleapis.com/storage/v1/b/bucket1\n" 841 "Auth Token: fake_token\n" 842 "Timeouts: 5 1 10\n", 843 "{\"size\": \"100\"}"), 844 new FakeHttpRequest( 845 "Uri: https://www.googleapis.com/storage/v1/b/bucket1\n" 846 "Auth Token: fake_token\n" 847 "Timeouts: 5 1 10\n", 848 "{\"size\": \"100\"}")}); 849 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 850 std::unique_ptr<HttpRequest::Factory>( 851 new FakeHttpRequestFactory(&requests)), 852 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 853 0 /* stat cache max age */, 0 /* stat cache max entries */, 854 0 /* matching paths cache max age */, 855 0 /* matching paths cache max entries */, 856 0 /* initial retry delay */, kTestTimeoutConfig, 857 nullptr /* gcs additional header */); 858 859 TF_EXPECT_OK(fs.FileExists("gs://bucket1")); 860 TF_EXPECT_OK(fs.FileExists("gs://bucket1/")); 861 } 862 863 TEST(GcsFileSystemTest, FileExists_NotAsObjectOrFolder) { 864 std::vector<HttpRequest*> requests( 865 {new FakeHttpRequest( 866 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 867 "path%2Ffile1.txt?fields=size%2Cupdated\n" 868 "Auth Token: fake_token\n" 869 "Timeouts: 5 1 10\n", 870 "", errors::NotFound("404"), 404), 871 new FakeHttpRequest( 872 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 873 "fields=items%2Fname%2CnextPageToken&prefix=path%2Ffile1.txt%2F" 874 "&maxResults=1\n" 875 "Auth Token: fake_token\n" 876 "Timeouts: 5 1 10\n", 877 "{\"items\": []}")}); 878 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 879 std::unique_ptr<HttpRequest::Factory>( 880 new FakeHttpRequestFactory(&requests)), 881 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 882 0 /* stat cache max age */, 0 /* stat cache max entries */, 883 0 /* matching paths cache max age */, 884 0 /* matching paths cache max entries */, 885 0 /* initial retry delay */, kTestTimeoutConfig, 886 nullptr /* gcs additional header */); 887 888 EXPECT_EQ(errors::Code::NOT_FOUND, 889 fs.FileExists("gs://bucket/path/file1.txt").code()); 890 } 891 892 TEST(GcsFileSystemTest, FileExists_NotAsBucket) { 893 std::vector<HttpRequest*> requests( 894 {new FakeHttpRequest( 895 "Uri: https://www.googleapis.com/storage/v1/b/bucket2\n" 896 "Auth Token: fake_token\n" 897 "Timeouts: 5 1 10\n", 898 "", errors::NotFound("404"), 404), 899 new FakeHttpRequest( 900 "Uri: https://www.googleapis.com/storage/v1/b/bucket2\n" 901 "Auth Token: fake_token\n" 902 "Timeouts: 5 1 10\n", 903 "", errors::NotFound("404"), 404)}); 904 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 905 std::unique_ptr<HttpRequest::Factory>( 906 new FakeHttpRequestFactory(&requests)), 907 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 908 0 /* stat cache max age */, 0 /* stat cache max entries */, 909 0 /* matching paths cache max age */, 910 0 /* matching paths cache max entries */, 911 0 /* initial retry delay */, kTestTimeoutConfig, 912 nullptr /* gcs additional header */); 913 EXPECT_EQ(errors::Code::INVALID_ARGUMENT, 914 fs.FileExists("gs://bucket2/").code()); 915 EXPECT_EQ(errors::Code::INVALID_ARGUMENT, 916 fs.FileExists("gs://bucket2").code()); 917 } 918 919 TEST(GcsFileSystemTest, FileExists_StatCache) { 920 std::vector<HttpRequest*> requests( 921 {new FakeHttpRequest( 922 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 923 "path%2Ffile1.txt?fields=size%2Cupdated\n" 924 "Auth Token: fake_token\n" 925 "Timeouts: 5 1 10\n", 926 strings::StrCat("{\"size\": \"1010\"," 927 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")), 928 new FakeHttpRequest( 929 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 930 "path%2Fsubfolder?fields=size%2Cupdated\n" 931 "Auth Token: fake_token\n" 932 "Timeouts: 5 1 10\n", 933 "", errors::NotFound("404"), 404), 934 new FakeHttpRequest( 935 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 936 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubfolder%2F" 937 "&maxResults=1\n" 938 "Auth Token: fake_token\n" 939 "Timeouts: 5 1 10\n", 940 "{\"items\": [ " 941 " { \"name\": \"path/subfolder/\" }]}")}); 942 GcsFileSystem fs( 943 std::unique_ptr<AuthProvider>(new FakeAuthProvider), 944 std::unique_ptr<HttpRequest::Factory>( 945 new FakeHttpRequestFactory(&requests)), 946 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 947 3600 /* stat cache max age */, 0 /* stat cache max entries */, 948 0 /* matching paths cache max age */, 949 0 /* matching paths cache max entries */, 0 /* initial retry delay */, 950 kTestTimeoutConfig, nullptr /* gcs additional header */); 951 952 // The stat cache will ensure that repeated lookups don't trigger additional 953 // HTTP requests. 954 for (int i = 0; i < 10; i++) { 955 TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt")); 956 TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder")); 957 } 958 } 959 960 TEST(GcsFileSystemTest, GetChildren_NoItems) { 961 std::vector<HttpRequest*> requests({new FakeHttpRequest( 962 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 963 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix=" 964 "path%2F\n" 965 "Auth Token: fake_token\n" 966 "Timeouts: 5 1 10\n", 967 "{\"prefixes\": [\"path/subpath/\"]}")}); 968 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 969 std::unique_ptr<HttpRequest::Factory>( 970 new FakeHttpRequestFactory(&requests)), 971 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 972 0 /* stat cache max age */, 0 /* stat cache max entries */, 973 0 /* matching paths cache max age */, 974 0 /* matching paths cache max entries */, 975 0 /* initial retry delay */, kTestTimeoutConfig, 976 nullptr /* gcs additional header */); 977 978 std::vector<string> children; 979 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); 980 981 EXPECT_EQ(std::vector<string>({"subpath/"}), children); 982 } 983 984 TEST(GcsFileSystemTest, GetChildren_ThreeFiles) { 985 std::vector<HttpRequest*> requests({new FakeHttpRequest( 986 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 987 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix=" 988 "path%2F\n" 989 "Auth Token: fake_token\n" 990 "Timeouts: 5 1 10\n", 991 "{\"items\": [ " 992 " { \"name\": \"path/file1.txt\" }," 993 " { \"name\": \"path/file3.txt\" }]," 994 "\"prefixes\": [\"path/subpath/\"]}")}); 995 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 996 std::unique_ptr<HttpRequest::Factory>( 997 new FakeHttpRequestFactory(&requests)), 998 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 999 0 /* stat cache max age */, 0 /* stat cache max entries */, 1000 0 /* matching paths cache max age */, 1001 0 /* matching paths cache max entries */, 1002 0 /* initial retry delay */, kTestTimeoutConfig, 1003 nullptr /* gcs additional header */); 1004 1005 std::vector<string> children; 1006 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); 1007 1008 EXPECT_EQ(std::vector<string>({"file1.txt", "file3.txt", "subpath/"}), 1009 children); 1010 } 1011 1012 TEST(GcsFileSystemTest, GetChildren_SelfDirectoryMarker) { 1013 std::vector<HttpRequest*> requests({new FakeHttpRequest( 1014 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1015 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix=" 1016 "path%2F\n" 1017 "Auth Token: fake_token\n" 1018 "Timeouts: 5 1 10\n", 1019 "{\"items\": [ " 1020 " { \"name\": \"path/\" }," 1021 " { \"name\": \"path/file3.txt\" }]," 1022 "\"prefixes\": [\"path/subpath/\"]}")}); 1023 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1024 std::unique_ptr<HttpRequest::Factory>( 1025 new FakeHttpRequestFactory(&requests)), 1026 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1027 0 /* stat cache max age */, 0 /* stat cache max entries */, 1028 0 /* matching paths cache max age */, 1029 0 /* matching paths cache max entries */, 1030 0 /* initial retry delay */, kTestTimeoutConfig, 1031 nullptr /* gcs additional header */); 1032 1033 std::vector<string> children; 1034 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); 1035 1036 EXPECT_EQ(std::vector<string>({"file3.txt", "subpath/"}), children); 1037 } 1038 1039 TEST(GcsFileSystemTest, GetChildren_ThreeFiles_NoSlash) { 1040 std::vector<HttpRequest*> requests({new FakeHttpRequest( 1041 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1042 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix=" 1043 "path%2F\n" 1044 "Auth Token: fake_token\n" 1045 "Timeouts: 5 1 10\n", 1046 "{\"items\": [ " 1047 " { \"name\": \"path/file1.txt\" }," 1048 " { \"name\": \"path/file3.txt\" }]," 1049 "\"prefixes\": [\"path/subpath/\"]}")}); 1050 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1051 std::unique_ptr<HttpRequest::Factory>( 1052 new FakeHttpRequestFactory(&requests)), 1053 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1054 0 /* stat cache max age */, 0 /* stat cache max entries */, 1055 0 /* matching paths cache max age */, 1056 0 /* matching paths cache max entries */, 1057 0 /* initial retry delay*/, kTestTimeoutConfig, 1058 nullptr /* gcs additional header */); 1059 1060 std::vector<string> children; 1061 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children)); 1062 1063 EXPECT_EQ(std::vector<string>({"file1.txt", "file3.txt", "subpath/"}), 1064 children); 1065 } 1066 1067 TEST(GcsFileSystemTest, GetChildren_Root) { 1068 std::vector<HttpRequest*> requests({new FakeHttpRequest( 1069 "Uri: https://www.googleapis.com/storage/v1/b/bucket-a-b-c/o?" 1070 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F\n" 1071 "Auth Token: fake_token\n" 1072 "Timeouts: 5 1 10\n", 1073 "{}")}); 1074 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1075 std::unique_ptr<HttpRequest::Factory>( 1076 new FakeHttpRequestFactory(&requests)), 1077 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1078 0 /* stat cache max age */, 0 /* stat cache max entries */, 1079 0 /* matching paths cache max age */, 1080 0 /* matching paths cache max entries */, 1081 0 /* initial retry delay*/, kTestTimeoutConfig, 1082 nullptr /* gcs additional header */); 1083 1084 std::vector<string> children; 1085 TF_EXPECT_OK(fs.GetChildren("gs://bucket-a-b-c", &children)); 1086 1087 EXPECT_EQ(0, children.size()); 1088 } 1089 1090 TEST(GcsFileSystemTest, GetChildren_Empty) { 1091 std::vector<HttpRequest*> requests({new FakeHttpRequest( 1092 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1093 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix=" 1094 "path%2F\n" 1095 "Auth Token: fake_token\n" 1096 "Timeouts: 5 1 10\n", 1097 "{}")}); 1098 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1099 std::unique_ptr<HttpRequest::Factory>( 1100 new FakeHttpRequestFactory(&requests)), 1101 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1102 0 /* stat cache max age */, 0 /* stat cache max entries */, 1103 0 /* matching paths cache max age */, 1104 0 /* matching paths cache max entries */, 1105 0 /* initial retry delay*/, kTestTimeoutConfig, 1106 nullptr /* gcs additional header */); 1107 1108 std::vector<string> children; 1109 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); 1110 1111 EXPECT_EQ(0, children.size()); 1112 } 1113 1114 TEST(GcsFileSystemTest, GetChildren_Pagination) { 1115 std::vector<HttpRequest*> requests( 1116 {new FakeHttpRequest( 1117 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1118 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&" 1119 "prefix=path%2F\n" 1120 "Auth Token: fake_token\n" 1121 "Timeouts: 5 1 10\n", 1122 "{\"nextPageToken\": \"ABCD==\", " 1123 "\"items\": [ " 1124 " { \"name\": \"path/file1.txt\" }," 1125 " { \"name\": \"path/file3.txt\" }]," 1126 "\"prefixes\": [\"path/subpath/\"]}"), 1127 new FakeHttpRequest( 1128 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1129 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&" 1130 "prefix=path%2F" 1131 "&pageToken=ABCD==\n" 1132 "Auth Token: fake_token\n" 1133 "Timeouts: 5 1 10\n", 1134 "{\"items\": [ " 1135 " { \"name\": \"path/file4.txt\" }," 1136 " { \"name\": \"path/file5.txt\" }]}")}); 1137 1138 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1139 std::unique_ptr<HttpRequest::Factory>( 1140 new FakeHttpRequestFactory(&requests)), 1141 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1142 0 /* stat cache max age */, 0 /* stat cache max entries */, 1143 0 /* matching paths cache max age */, 1144 0 /* matching paths cache max entries */, 1145 0 /* initial retry delay*/, kTestTimeoutConfig, 1146 nullptr /* gcs additional header */); 1147 1148 std::vector<string> children; 1149 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children)); 1150 1151 EXPECT_EQ(std::vector<string>({"file1.txt", "file3.txt", "subpath/", 1152 "file4.txt", "file5.txt"}), 1153 children); 1154 } 1155 1156 TEST(GcsFileSystemTest, GetMatchingPaths_NoWildcard) { 1157 std::vector<HttpRequest*> requests({new FakeHttpRequest( 1158 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1159 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n" 1160 "Auth Token: fake_token\n" 1161 "Timeouts: 5 1 10\n", 1162 "{\"items\": [ " 1163 " { \"name\": \"path/subpath/file2.txt\" }]}")}); 1164 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1165 std::unique_ptr<HttpRequest::Factory>( 1166 new FakeHttpRequestFactory(&requests)), 1167 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1168 0 /* stat cache max age */, 0 /* stat cache max entries */, 1169 0 /* matching paths cache max age */, 1170 0 /* matching paths cache max entries */, 1171 0 /* initial retry delay*/, kTestTimeoutConfig, 1172 nullptr /* gcs additional header */); 1173 1174 std::vector<string> result; 1175 TF_EXPECT_OK( 1176 fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt", &result)); 1177 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}), 1178 result); 1179 } 1180 1181 TEST(GcsFileSystemTest, GetMatchingPaths_BucketAndWildcard) { 1182 std::vector<HttpRequest*> requests({new FakeHttpRequest( 1183 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1184 "fields=items%2Fname%2CnextPageToken\n" 1185 "Auth Token: fake_token\n" 1186 "Timeouts: 5 1 10\n", 1187 "{\"items\": [ " 1188 " { \"name\": \"path/file1.txt\" }," 1189 " { \"name\": \"path/subpath/file2.txt\" }," 1190 " { \"name\": \"path/file3.txt\" }]}")}); 1191 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1192 std::unique_ptr<HttpRequest::Factory>( 1193 new FakeHttpRequestFactory(&requests)), 1194 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1195 0 /* stat cache max age */, 0 /* stat cache max entries */, 1196 0 /* matching paths cache max age */, 1197 0 /* matching paths cache max entries */, 1198 0 /* initial retry delay*/, kTestTimeoutConfig, 1199 nullptr /* gcs additional header */); 1200 1201 std::vector<string> result; 1202 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", &result)); 1203 EXPECT_EQ(std::vector<string>({"gs://bucket/path/file1.txt", 1204 "gs://bucket/path/file3.txt", 1205 "gs://bucket/path/subpath"}), 1206 result); 1207 } 1208 1209 TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_Matches) { 1210 std::vector<HttpRequest*> requests({new FakeHttpRequest( 1211 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1212 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n" 1213 "Auth Token: fake_token\n" 1214 "Timeouts: 5 1 10\n", 1215 "{\"items\": [ " 1216 " { \"name\": \"path/file1.txt\" }," 1217 " { \"name\": \"path/subpath/file2.txt\" }," 1218 " { \"name\": \"path/file3.txt\" }]}")}); 1219 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1220 std::unique_ptr<HttpRequest::Factory>( 1221 new FakeHttpRequestFactory(&requests)), 1222 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1223 0 /* stat cache max age */, 0 /* stat cache max entries */, 1224 0 /* matching paths cache max age */, 1225 0 /* matching paths cache max entries */, 1226 0 /* initial retry delay*/, kTestTimeoutConfig, 1227 nullptr /* gcs additional header */); 1228 1229 std::vector<string> result; 1230 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file2.txt", &result)); 1231 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}), 1232 result); 1233 } 1234 1235 TEST(GcsFileSystemTest, GetMatchingPaths_SelfDirectoryMarker) { 1236 std::vector<HttpRequest*> requests({new FakeHttpRequest( 1237 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1238 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n" 1239 "Auth Token: fake_token\n" 1240 "Timeouts: 5 1 10\n", 1241 "{\"items\": [ " 1242 " { \"name\": \"path/\" }," 1243 " { \"name\": \"path/file3.txt\" }]}")}); 1244 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1245 std::unique_ptr<HttpRequest::Factory>( 1246 new FakeHttpRequestFactory(&requests)), 1247 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1248 0 /* stat cache max age */, 0 /* stat cache max entries */, 1249 0 /* matching paths cache max age */, 1250 0 /* matching paths cache max entries */, 1251 0 /* initial retry delay*/, kTestTimeoutConfig, 1252 nullptr /* gcs additional header */); 1253 1254 std::vector<string> result; 1255 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*", &result)); 1256 EXPECT_EQ(std::vector<string>({"gs://bucket/path/file3.txt"}), result); 1257 } 1258 1259 TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_NoMatches) { 1260 std::vector<HttpRequest*> requests({new FakeHttpRequest( 1261 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1262 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n" 1263 "Auth Token: fake_token\n" 1264 "Timeouts: 5 1 10\n", 1265 "{\"items\": [ " 1266 " { \"name\": \"path/file1.txt\" }," 1267 " { \"name\": \"path/subpath/file2.txt\" }," 1268 " { \"name\": \"path/file3.txt\" }]}")}); 1269 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1270 std::unique_ptr<HttpRequest::Factory>( 1271 new FakeHttpRequestFactory(&requests)), 1272 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1273 0 /* stat cache max age */, 0 /* stat cache max entries */, 1274 0 /* matching paths cache max age */, 1275 0 /* matching paths cache max entries */, 1276 0 /* initial retry delay*/, kTestTimeoutConfig, 1277 nullptr /* gcs additional header */); 1278 1279 std::vector<string> result; 1280 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file3.txt", &result)); 1281 EXPECT_EQ(std::vector<string>(), result); 1282 } 1283 1284 TEST(GcsFileSystemTest, GetMatchingPaths_OnlyWildcard) { 1285 std::vector<HttpRequest*> requests; 1286 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1287 std::unique_ptr<HttpRequest::Factory>( 1288 new FakeHttpRequestFactory(&requests)), 1289 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1290 0 /* stat cache max age */, 0 /* stat cache max entries */, 1291 0 /* matching paths cache max age */, 1292 0 /* matching paths cache max entries */, 1293 0 /* initial retry delay*/, kTestTimeoutConfig, 1294 nullptr /* gcs additional header */); 1295 1296 std::vector<string> result; 1297 EXPECT_EQ(errors::Code::INVALID_ARGUMENT, 1298 fs.GetMatchingPaths("gs://*", &result).code()); 1299 } 1300 1301 TEST(GcsFileSystemTest, GetMatchingPaths_Cache) { 1302 std::vector<HttpRequest*> requests( 1303 {new FakeHttpRequest( 1304 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1305 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n" 1306 "Auth Token: fake_token\n" 1307 "Timeouts: 5 1 10\n", 1308 "{\"items\": [ " 1309 " { \"name\": \"path/subpath/file2.txt\" }]}"), 1310 new FakeHttpRequest( 1311 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1312 "fields=items%2Fname%2CnextPageToken\n" 1313 "Auth Token: fake_token\n" 1314 "Timeouts: 5 1 10\n", 1315 "{\"items\": [ " 1316 " { \"name\": \"path/file1.txt\" }," 1317 " { \"name\": \"path/subpath/file2.txt\" }," 1318 " { \"name\": \"path/file3.txt\" }]}")}); 1319 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1320 std::unique_ptr<HttpRequest::Factory>( 1321 new FakeHttpRequestFactory(&requests)), 1322 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1323 0 /* stat cache max age */, 0 /* stat cache max entries */, 1324 3600 /* matching paths cache max age */, 1325 0 /* matching paths cache max entries */, 1326 0 /* initial retry delay*/, kTestTimeoutConfig, 1327 nullptr /* gcs additional header */); 1328 1329 // Repeated calls to fs.GetMatchingPaths on these patterns should not lead to 1330 // any additional HTTP requests to GCS. 1331 for (int i = 0; i < 10; i++) { 1332 std::vector<string> result; 1333 TF_EXPECT_OK( 1334 fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt", &result)); 1335 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}), 1336 result); 1337 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", &result)); 1338 EXPECT_EQ(std::vector<string>({"gs://bucket/path/file1.txt", 1339 "gs://bucket/path/file3.txt", 1340 "gs://bucket/path/subpath"}), 1341 result); 1342 } 1343 } 1344 1345 TEST(GcsFileSystemTest, GetMatchingPaths_Cache_Flush) { 1346 std::vector<HttpRequest*> requests( 1347 {new FakeHttpRequest( 1348 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1349 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n" 1350 "Auth Token: fake_token\n" 1351 "Timeouts: 5 1 10\n", 1352 "{\"items\": [ " 1353 " { \"name\": \"path/subpath/file2.txt\" }]}"), 1354 new FakeHttpRequest( 1355 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1356 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n" 1357 "Auth Token: fake_token\n" 1358 "Timeouts: 5 1 10\n", 1359 "{\"items\": [ " 1360 " { \"name\": \"path/subpath/file2.txt\" }]}")}); 1361 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1362 std::unique_ptr<HttpRequest::Factory>( 1363 new FakeHttpRequestFactory(&requests)), 1364 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1365 0 /* stat cache max age */, 0 /* stat cache max entries */, 1366 3600 /* matching paths cache max age */, 1367 0 /* matching paths cache max entries */, 1368 0 /* initial retry delay*/, kTestTimeoutConfig, 1369 nullptr /* gcs additional header */); 1370 1371 // This loop should trigger the first HTTP request to GCS. 1372 for (int i = 0; i < 10; i++) { 1373 std::vector<string> result; 1374 TF_EXPECT_OK( 1375 fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt", &result)); 1376 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}), 1377 result); 1378 } 1379 // After flushing caches, there should be another (identical) request to GCS. 1380 fs.FlushCaches(); 1381 for (int i = 0; i < 10; i++) { 1382 std::vector<string> result; 1383 TF_EXPECT_OK( 1384 fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt", &result)); 1385 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}), 1386 result); 1387 } 1388 } 1389 1390 TEST(GcsFileSystemTest, DeleteFile) { 1391 std::vector<HttpRequest*> requests( 1392 {new FakeHttpRequest( 1393 "Uri: https://storage.googleapis.com/bucket/path%2Ffile1.txt\n" 1394 "Auth Token: fake_token\n" 1395 "Range: 0-15\n" 1396 "Timeouts: 5 1 20\n", 1397 "01234567"), 1398 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" 1399 "/bucket/o/path%2Ffile1.txt\n" 1400 "Auth Token: fake_token\n" 1401 "Timeouts: 5 1 10\n" 1402 "Delete: yes\n", 1403 ""), 1404 new FakeHttpRequest( 1405 "Uri: https://storage.googleapis.com/bucket/path%2Ffile1.txt\n" 1406 "Auth Token: fake_token\n" 1407 "Range: 0-15\n" 1408 "Timeouts: 5 1 20\n", 1409 "76543210")}); 1410 GcsFileSystem fs( 1411 std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1412 std::unique_ptr<HttpRequest::Factory>( 1413 new FakeHttpRequestFactory(&requests)), 1414 16 /* block size */, 16 /* max bytes */, 0 /* max staleness */, 1415 0 /* stat cache max age */, 0 /* stat cache max entries */, 1416 0 /* matching paths cache max age */, 1417 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, 1418 kTestTimeoutConfig, nullptr /* gcs additional header */); 1419 1420 // Do an initial read of the file to load its contents into the block cache. 1421 char scratch[100]; 1422 StringPiece result; 1423 std::unique_ptr<RandomAccessFile> file; 1424 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/file1.txt", &file)); 1425 TF_EXPECT_OK(file->Read(0, 8, &result, scratch)); 1426 EXPECT_EQ("01234567", result); 1427 // Deleting the file triggers the next HTTP request to GCS. 1428 TF_EXPECT_OK(fs.DeleteFile("gs://bucket/path/file1.txt")); 1429 // Re-reading the file causes its contents to be reloaded from GCS and not 1430 // from the block cache. 1431 TF_EXPECT_OK(file->Read(0, 8, &result, scratch)); 1432 EXPECT_EQ("76543210", result); 1433 } 1434 1435 TEST(GcsFileSystemTest, DeleteFile_NoObjectName) { 1436 std::vector<HttpRequest*> requests; 1437 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1438 std::unique_ptr<HttpRequest::Factory>( 1439 new FakeHttpRequestFactory(&requests)), 1440 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1441 0 /* stat cache max age */, 0 /* stat cache max entries */, 1442 0 /* matching paths cache max age */, 1443 0 /* matching paths cache max entries */, 1444 0 /* initial retry delay*/, kTestTimeoutConfig, 1445 nullptr /* gcs additional header */); 1446 1447 EXPECT_EQ(errors::Code::INVALID_ARGUMENT, 1448 fs.DeleteFile("gs://bucket/").code()); 1449 } 1450 1451 TEST(GcsFileSystemTest, DeleteDir_Empty) { 1452 std::vector<HttpRequest*> requests({new FakeHttpRequest( 1453 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1454 "fields=items%2Fname%2CnextPageToken&prefix=path%2F&maxResults=2\n" 1455 "Auth Token: fake_token\n" 1456 "Timeouts: 5 1 10\n", 1457 "{}")}); 1458 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1459 std::unique_ptr<HttpRequest::Factory>( 1460 new FakeHttpRequestFactory(&requests)), 1461 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1462 0 /* stat cache max age */, 0 /* stat cache max entries */, 1463 0 /* matching paths cache max age */, 1464 0 /* matching paths cache max entries */, 1465 0 /* initial retry delay*/, kTestTimeoutConfig, 1466 nullptr /* gcs additional header */); 1467 1468 TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/")); 1469 } 1470 1471 TEST(GcsFileSystemTest, DeleteDir_OnlyDirMarkerLeft) { 1472 std::vector<HttpRequest*> requests( 1473 {new FakeHttpRequest( 1474 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1475 "fields=items%2Fname%2CnextPageToken&prefix=path%2F&maxResults=2\n" 1476 "Auth Token: fake_token\n" 1477 "Timeouts: 5 1 10\n", 1478 "{\"items\": [ " 1479 " { \"name\": \"path/\" }]}"), 1480 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" 1481 "/bucket/o/path%2F\n" 1482 "Auth Token: fake_token\n" 1483 "Timeouts: 5 1 10\n" 1484 "Delete: yes\n", 1485 "")}); 1486 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1487 std::unique_ptr<HttpRequest::Factory>( 1488 new FakeHttpRequestFactory(&requests)), 1489 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1490 0 /* stat cache max age */, 0 /* stat cache max entries */, 1491 0 /* matching paths cache max age */, 1492 0 /* matching paths cache max entries */, 1493 0 /* initial retry delay*/, kTestTimeoutConfig, 1494 nullptr /* gcs additional header */); 1495 1496 TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/")); 1497 } 1498 1499 TEST(GcsFileSystemTest, DeleteDir_BucketOnly) { 1500 std::vector<HttpRequest*> requests({new FakeHttpRequest( 1501 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?fields=items%2F" 1502 "name%2CnextPageToken&maxResults=2\nAuth Token: fake_token\n" 1503 "Timeouts: 5 1 10\n", 1504 "{}")}); 1505 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1506 std::unique_ptr<HttpRequest::Factory>( 1507 new FakeHttpRequestFactory(&requests)), 1508 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1509 0 /* stat cache max age */, 0 /* stat cache max entries */, 1510 0 /* matching paths cache max age */, 1511 0 /* matching paths cache max entries */, 1512 0 /* initial retry delay*/, kTestTimeoutConfig, 1513 nullptr /* gcs additional header */); 1514 1515 TF_EXPECT_OK(fs.DeleteDir("gs://bucket")); 1516 } 1517 1518 TEST(GcsFileSystemTest, DeleteDir_NonEmpty) { 1519 std::vector<HttpRequest*> requests({new FakeHttpRequest( 1520 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1521 "fields=items%2Fname%2CnextPageToken&prefix=path%2F&maxResults=2\n" 1522 "Auth Token: fake_token\n" 1523 "Timeouts: 5 1 10\n", 1524 "{\"items\": [ " 1525 " { \"name\": \"path/file1.txt\" }]}")}); 1526 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1527 std::unique_ptr<HttpRequest::Factory>( 1528 new FakeHttpRequestFactory(&requests)), 1529 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1530 0 /* stat cache max age */, 0 /* stat cache max entries */, 1531 0 /* matching paths cache max age */, 1532 0 /* matching paths cache max entries */, 1533 0 /* initial retry delay*/, kTestTimeoutConfig, 1534 nullptr /* gcs additional header */); 1535 1536 EXPECT_EQ(error::Code::FAILED_PRECONDITION, 1537 fs.DeleteDir("gs://bucket/path/").code()); 1538 } 1539 1540 TEST(GcsFileSystemTest, GetFileSize) { 1541 std::vector<HttpRequest*> requests({new FakeHttpRequest( 1542 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1543 "file.txt?fields=size%2Cupdated\n" 1544 "Auth Token: fake_token\n" 1545 "Timeouts: 5 1 10\n", 1546 strings::StrCat("{\"size\": \"1010\"," 1547 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); 1548 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1549 std::unique_ptr<HttpRequest::Factory>( 1550 new FakeHttpRequestFactory(&requests)), 1551 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1552 0 /* stat cache max age */, 0 /* stat cache max entries */, 1553 0 /* matching paths cache max age */, 1554 0 /* matching paths cache max entries */, 1555 0 /* initial retry delay*/, kTestTimeoutConfig, 1556 nullptr /* gcs additional header */); 1557 1558 uint64 size; 1559 TF_EXPECT_OK(fs.GetFileSize("gs://bucket/file.txt", &size)); 1560 EXPECT_EQ(1010, size); 1561 } 1562 1563 TEST(GcsFileSystemTest, GetFileSize_NoObjectName) { 1564 std::vector<HttpRequest*> requests; 1565 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1566 std::unique_ptr<HttpRequest::Factory>( 1567 new FakeHttpRequestFactory(&requests)), 1568 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1569 0 /* stat cache max age */, 0 /* stat cache max entries */, 1570 0 /* matching paths cache max age */, 1571 0 /* matching paths cache max entries */, 1572 0 /* initial retry delay*/, kTestTimeoutConfig, 1573 nullptr /* gcs additional header */); 1574 1575 uint64 size; 1576 EXPECT_EQ(errors::Code::INVALID_ARGUMENT, 1577 fs.GetFileSize("gs://bucket/", &size).code()); 1578 } 1579 1580 TEST(GcsFileSystemTest, RenameFile_Folder) { 1581 std::vector<HttpRequest*> requests( 1582 {// Check if this is a folder or an object. 1583 new FakeHttpRequest( 1584 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1585 "fields=items%2Fname%2CnextPageToken&prefix=path1%2F" 1586 "&maxResults=1\n" 1587 "Auth Token: fake_token\n" 1588 "Timeouts: 5 1 10\n", 1589 "{\"items\": [ " 1590 " { \"name\": \"path1/subfolder/file1.txt\" }]}"), 1591 // Requesting the full list of files in the folder. 1592 new FakeHttpRequest( 1593 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1594 "fields=items%2Fname%2CnextPageToken&prefix=path1%2F\n" 1595 "Auth Token: fake_token\n" 1596 "Timeouts: 5 1 10\n", 1597 "{\"items\": [ " 1598 " { \"name\": \"path1/\" }," // A directory marker. 1599 " { \"name\": \"path1/subfolder/file1.txt\" }," 1600 " { \"name\": \"path1/file2.txt\" }]}"), 1601 // Copying the directory marker. 1602 new FakeHttpRequest( 1603 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1604 "path1%2F/rewriteTo/b/bucket/o/path2%2F\n" 1605 "Auth Token: fake_token\n" 1606 "Post: yes\n" 1607 "Timeouts: 5 1 10\n", 1608 "{\"done\": true}"), 1609 // Deleting the original directory marker. 1610 new FakeHttpRequest( 1611 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1612 "path1%2F\n" 1613 "Auth Token: fake_token\n" 1614 "Timeouts: 5 1 10\n" 1615 "Delete: yes\n", 1616 ""), 1617 // Copying the first file. 1618 new FakeHttpRequest( 1619 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1620 "path1%2Fsubfolder%2Ffile1.txt/rewriteTo/b/bucket/o/" 1621 "path2%2Fsubfolder%2Ffile1.txt\n" 1622 "Auth Token: fake_token\n" 1623 "Post: yes\n" 1624 "Timeouts: 5 1 10\n", 1625 "{\"done\": true}"), 1626 // Deleting the first original file. 1627 new FakeHttpRequest( 1628 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1629 "path1%2Fsubfolder%2Ffile1.txt\n" 1630 "Auth Token: fake_token\n" 1631 "Timeouts: 5 1 10\n" 1632 "Delete: yes\n", 1633 ""), 1634 // Copying the second file. 1635 new FakeHttpRequest( 1636 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1637 "path1%2Ffile2.txt/rewriteTo/b/bucket/o/path2%2Ffile2.txt\n" 1638 "Auth Token: fake_token\n" 1639 "Post: yes\n" 1640 "Timeouts: 5 1 10\n", 1641 "{\"done\": true}"), 1642 // Deleting the second original file. 1643 new FakeHttpRequest( 1644 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1645 "path1%2Ffile2.txt\n" 1646 "Auth Token: fake_token\n" 1647 "Timeouts: 5 1 10\n" 1648 "Delete: yes\n", 1649 "")}); 1650 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1651 std::unique_ptr<HttpRequest::Factory>( 1652 new FakeHttpRequestFactory(&requests)), 1653 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1654 0 /* stat cache max age */, 0 /* stat cache max entries */, 1655 0 /* matching paths cache max age */, 1656 0 /* matching paths cache max entries */, 1657 0 /* initial retry delay*/, kTestTimeoutConfig, 1658 nullptr /* gcs additional header */); 1659 1660 TF_EXPECT_OK(fs.RenameFile("gs://bucket/path1", "gs://bucket/path2/")); 1661 } 1662 1663 TEST(GcsFileSystemTest, RenameFile_Object) { 1664 std::vector<HttpRequest*> requests( 1665 {new FakeHttpRequest( 1666 "Uri: https://storage.googleapis.com/bucket/path%2Fsrc.txt\n" 1667 "Auth Token: fake_token\n" 1668 "Range: 0-15\n" 1669 "Timeouts: 5 1 20\n", 1670 "01234567"), 1671 new FakeHttpRequest( 1672 "Uri: https://storage.googleapis.com/bucket/path%2Fdst.txt\n" 1673 "Auth Token: fake_token\n" 1674 "Range: 0-15\n" 1675 "Timeouts: 5 1 20\n", 1676 "76543210"), 1677 // IsDirectory is checking whether there are children objects. 1678 new FakeHttpRequest( 1679 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1680 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F" 1681 "&maxResults=1\n" 1682 "Auth Token: fake_token\n" 1683 "Timeouts: 5 1 10\n", 1684 "{}"), 1685 // IsDirectory is checking if the path exists as an object. 1686 new FakeHttpRequest( 1687 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1688 "path%2Fsrc.txt?fields=size%2Cupdated\n" 1689 "Auth Token: fake_token\n" 1690 "Timeouts: 5 1 10\n", 1691 strings::StrCat("{\"size\": \"1010\"," 1692 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")), 1693 // Copying to the new location. 1694 new FakeHttpRequest( 1695 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1696 "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n" 1697 "Auth Token: fake_token\n" 1698 "Post: yes\n" 1699 "Timeouts: 5 1 10\n", 1700 "{\"done\": true}"), 1701 // Deleting the original file. 1702 new FakeHttpRequest( 1703 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1704 "path%2Fsrc.txt\n" 1705 "Auth Token: fake_token\n" 1706 "Timeouts: 5 1 10\n" 1707 "Delete: yes\n", 1708 ""), 1709 new FakeHttpRequest( 1710 "Uri: https://storage.googleapis.com/bucket/path%2Fsrc.txt\n" 1711 "Auth Token: fake_token\n" 1712 "Range: 0-15\n" 1713 "Timeouts: 5 1 20\n", 1714 "89abcdef"), 1715 new FakeHttpRequest( 1716 "Uri: https://storage.googleapis.com/bucket/path%2Fdst.txt\n" 1717 "Auth Token: fake_token\n" 1718 "Range: 0-15\n" 1719 "Timeouts: 5 1 20\n", 1720 "fedcba98")}); 1721 GcsFileSystem fs( 1722 std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1723 std::unique_ptr<HttpRequest::Factory>( 1724 new FakeHttpRequestFactory(&requests)), 1725 16 /* block size */, 64 /* max bytes */, 0 /* max staleness */, 1726 0 /* stat cache max age */, 0 /* stat cache max entries */, 1727 0 /* matching paths cache max age */, 1728 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, 1729 kTestTimeoutConfig, nullptr /* gcs additional header */); 1730 // Do an initial read of the source and destination files to load their 1731 // contents into the block cache. 1732 char scratch[100]; 1733 StringPiece result; 1734 std::unique_ptr<RandomAccessFile> src; 1735 std::unique_ptr<RandomAccessFile> dst; 1736 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/src.txt", &src)); 1737 TF_EXPECT_OK(src->Read(0, 8, &result, scratch)); 1738 EXPECT_EQ("01234567", result); 1739 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/dst.txt", &dst)); 1740 TF_EXPECT_OK(dst->Read(0, 8, &result, scratch)); 1741 EXPECT_EQ("76543210", result); 1742 // Now rename src to dst. This should flush the block cache for both files. 1743 TF_EXPECT_OK( 1744 fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt")); 1745 // Re-read both files. This should reload their contents from GCS. 1746 TF_EXPECT_OK(src->Read(0, 8, &result, scratch)); 1747 EXPECT_EQ("89abcdef", result); 1748 TF_EXPECT_OK(dst->Read(0, 8, &result, scratch)); 1749 EXPECT_EQ("fedcba98", result); 1750 } 1751 1752 /// Tests the scenario when deletion returns a failure, but actually succeeds. 1753 TEST(GcsFileSystemTest, RenameFile_Object_DeletionRetried) { 1754 std::vector<HttpRequest*> requests( 1755 {// IsDirectory is checking whether there are children objects. 1756 new FakeHttpRequest( 1757 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1758 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F" 1759 "&maxResults=1\n" 1760 "Auth Token: fake_token\n" 1761 "Timeouts: 5 1 10\n", 1762 "{}"), 1763 // IsDirectory is checking if the path exists as an object. 1764 new FakeHttpRequest( 1765 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1766 "path%2Fsrc.txt?fields=size%2Cupdated\n" 1767 "Auth Token: fake_token\n" 1768 "Timeouts: 5 1 10\n", 1769 strings::StrCat("{\"size\": \"1010\"," 1770 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")), 1771 // Copying to the new location. 1772 new FakeHttpRequest( 1773 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1774 "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n" 1775 "Auth Token: fake_token\n" 1776 "Post: yes\n" 1777 "Timeouts: 5 1 10\n", 1778 "{\"done\": true}"), 1779 // Deleting the original file - the deletion returns a failure. 1780 new FakeHttpRequest( 1781 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1782 "path%2Fsrc.txt\n" 1783 "Auth Token: fake_token\n" 1784 "Timeouts: 5 1 10\n" 1785 "Delete: yes\n", 1786 "", errors::Unavailable("503"), 503), 1787 // Deleting the original file again - the deletion returns NOT_FOUND. 1788 new FakeHttpRequest( 1789 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1790 "path%2Fsrc.txt\n" 1791 "Auth Token: fake_token\n" 1792 "Timeouts: 5 1 10\n" 1793 "Delete: yes\n", 1794 "", errors::NotFound("404"), 404)}); 1795 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1796 std::unique_ptr<HttpRequest::Factory>( 1797 new FakeHttpRequestFactory(&requests)), 1798 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1799 0 /* stat cache max age */, 0 /* stat cache max entries */, 1800 0 /* matching paths cache max age */, 1801 0 /* matching paths cache max entries */, 1802 0 /* initial retry delay*/, kTestTimeoutConfig, 1803 nullptr /* gcs additional header */); 1804 1805 TF_EXPECT_OK( 1806 fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt")); 1807 } 1808 1809 /// Tests the case when rewrite couldn't complete in one RPC. 1810 TEST(GcsFileSystemTest, RenameFile_Object_Incomplete) { 1811 std::vector<HttpRequest*> requests( 1812 {// IsDirectory is checking whether there are children objects. 1813 new FakeHttpRequest( 1814 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1815 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F" 1816 "&maxResults=1\n" 1817 "Auth Token: fake_token\n" 1818 "Timeouts: 5 1 10\n", 1819 "{}"), 1820 // IsDirectory is checking if the path exists as an object. 1821 new FakeHttpRequest( 1822 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1823 "path%2Fsrc.txt?fields=size%2Cupdated\n" 1824 "Auth Token: fake_token\n" 1825 "Timeouts: 5 1 10\n", 1826 strings::StrCat("{\"size\": \"1010\"," 1827 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")), 1828 // Copying to the new location. 1829 new FakeHttpRequest( 1830 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1831 "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n" 1832 "Auth Token: fake_token\n" 1833 "Post: yes\n" 1834 "Timeouts: 5 1 10\n", 1835 "{\"done\": false}")}); 1836 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1837 std::unique_ptr<HttpRequest::Factory>( 1838 new FakeHttpRequestFactory(&requests)), 1839 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1840 0 /* stat cache max age */, 0 /* stat cache max entries */, 1841 0 /* matching paths cache max age */, 1842 0 /* matching paths cache max entries */, 1843 0 /* initial retry delay*/, kTestTimeoutConfig, 1844 nullptr /* gcs additional header */); 1845 1846 EXPECT_EQ( 1847 errors::Code::UNIMPLEMENTED, 1848 fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt") 1849 .code()); 1850 } 1851 1852 TEST(GcsFileSystemTest, Stat_Object) { 1853 std::vector<HttpRequest*> requests({new FakeHttpRequest( 1854 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1855 "file.txt?fields=size%2Cupdated\n" 1856 "Auth Token: fake_token\n" 1857 "Timeouts: 5 1 10\n", 1858 strings::StrCat("{\"size\": \"1010\"," 1859 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); 1860 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1861 std::unique_ptr<HttpRequest::Factory>( 1862 new FakeHttpRequestFactory(&requests)), 1863 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1864 0 /* stat cache max age */, 0 /* stat cache max entries */, 1865 0 /* matching paths cache max age */, 1866 0 /* matching paths cache max entries */, 1867 0 /* initial retry delay*/, kTestTimeoutConfig, 1868 nullptr /* gcs additional header */); 1869 1870 FileStatistics stat; 1871 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat)); 1872 EXPECT_EQ(1010, stat.length); 1873 EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1); 1874 EXPECT_FALSE(stat.is_directory); 1875 } 1876 1877 TEST(GcsFileSystemTest, Stat_Folder) { 1878 std::vector<HttpRequest*> requests( 1879 {new FakeHttpRequest( 1880 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1881 "subfolder?fields=size%2Cupdated\n" 1882 "Auth Token: fake_token\n" 1883 "Timeouts: 5 1 10\n", 1884 "", errors::NotFound("404"), 404), 1885 new FakeHttpRequest( 1886 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1887 "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F" 1888 "&maxResults=1\n" 1889 "Auth Token: fake_token\n" 1890 "Timeouts: 5 1 10\n", 1891 "{\"items\": [ " 1892 " { \"name\": \"subfolder/\" }]}")}); 1893 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1894 std::unique_ptr<HttpRequest::Factory>( 1895 new FakeHttpRequestFactory(&requests)), 1896 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1897 0 /* stat cache max age */, 0 /* stat cache max entries */, 1898 0 /* matching paths cache max age */, 1899 0 /* matching paths cache max entries */, 1900 0 /* initial retry delay*/, kTestTimeoutConfig, 1901 nullptr /* gcs additional header */); 1902 1903 FileStatistics stat; 1904 TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder", &stat)); 1905 EXPECT_EQ(0, stat.length); 1906 EXPECT_EQ(0, stat.mtime_nsec); 1907 EXPECT_TRUE(stat.is_directory); 1908 } 1909 1910 TEST(GcsFileSystemTest, Stat_ObjectOrFolderNotFound) { 1911 std::vector<HttpRequest*> requests( 1912 {new FakeHttpRequest( 1913 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1914 "path?fields=size%2Cupdated\n" 1915 "Auth Token: fake_token\n" 1916 "Timeouts: 5 1 10\n", 1917 "", errors::NotFound("404"), 404), 1918 new FakeHttpRequest( 1919 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1920 "fields=items%2Fname%2CnextPageToken&prefix=path%2F" 1921 "&maxResults=1\n" 1922 "Auth Token: fake_token\n" 1923 "Timeouts: 5 1 10\n", 1924 "{}")}); 1925 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1926 std::unique_ptr<HttpRequest::Factory>( 1927 new FakeHttpRequestFactory(&requests)), 1928 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1929 0 /* stat cache max age */, 0 /* stat cache max entries */, 1930 0 /* matching paths cache max age */, 1931 0 /* matching paths cache max entries */, 1932 0 /* initial retry delay*/, kTestTimeoutConfig, 1933 nullptr /* gcs additional header */); 1934 1935 FileStatistics stat; 1936 EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/path", &stat).code()); 1937 } 1938 1939 TEST(GcsFileSystemTest, Stat_Bucket) { 1940 std::vector<HttpRequest*> requests({new FakeHttpRequest( 1941 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n" 1942 "Auth Token: fake_token\n" 1943 "Timeouts: 5 1 10\n", 1944 "{}")}); 1945 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1946 std::unique_ptr<HttpRequest::Factory>( 1947 new FakeHttpRequestFactory(&requests)), 1948 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1949 0 /* stat cache max age */, 0 /* stat cache max entries */, 1950 0 /* matching paths cache max age */, 1951 0 /* matching paths cache max entries */, 1952 0 /* initial retry delay*/, kTestTimeoutConfig, 1953 nullptr /* gcs additional header */); 1954 1955 FileStatistics stat; 1956 TF_EXPECT_OK(fs.Stat("gs://bucket/", &stat)); 1957 EXPECT_EQ(0, stat.length); 1958 EXPECT_EQ(0, stat.mtime_nsec); 1959 EXPECT_TRUE(stat.is_directory); 1960 } 1961 1962 TEST(GcsFileSystemTest, Stat_BucketNotFound) { 1963 std::vector<HttpRequest*> requests({new FakeHttpRequest( 1964 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n" 1965 "Auth Token: fake_token\n" 1966 "Timeouts: 5 1 10\n", 1967 "", errors::NotFound("404"), 404)}); 1968 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 1969 std::unique_ptr<HttpRequest::Factory>( 1970 new FakeHttpRequestFactory(&requests)), 1971 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 1972 0 /* stat cache max age */, 0 /* stat cache max entries */, 1973 0 /* matching paths cache max age */, 1974 0 /* matching paths cache max entries */, 1975 0 /* initial retry delay*/, kTestTimeoutConfig, 1976 nullptr /* gcs additional header */); 1977 1978 FileStatistics stat; 1979 EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/", &stat).code()); 1980 } 1981 1982 TEST(GcsFileSystemTest, Stat_Cache) { 1983 std::vector<HttpRequest*> requests( 1984 {new FakeHttpRequest( 1985 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1986 "file.txt?fields=size%2Cupdated\n" 1987 "Auth Token: fake_token\n" 1988 "Timeouts: 5 1 10\n", 1989 strings::StrCat("{\"size\": \"1010\"," 1990 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")), 1991 new FakeHttpRequest( 1992 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 1993 "subfolder?fields=size%2Cupdated\n" 1994 "Auth Token: fake_token\n" 1995 "Timeouts: 5 1 10\n", 1996 "", errors::NotFound("404"), 404), 1997 new FakeHttpRequest( 1998 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 1999 "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F" 2000 "&maxResults=1\n" 2001 "Auth Token: fake_token\n" 2002 "Timeouts: 5 1 10\n", 2003 "{\"items\": [ " 2004 " { \"name\": \"subfolder/\" }]}")}); 2005 GcsFileSystem fs( 2006 std::unique_ptr<AuthProvider>(new FakeAuthProvider), 2007 std::unique_ptr<HttpRequest::Factory>( 2008 new FakeHttpRequestFactory(&requests)), 2009 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 2010 3600 /* stat cache max age */, 0 /* stat cache max entries */, 2011 0 /* matching paths cache max age */, 2012 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, 2013 kTestTimeoutConfig, nullptr /* gcs additional header */); 2014 2015 // Repeated calls to fs.Stat on these paths should not lead to any additional 2016 // HTTP requests to GCS. 2017 for (int i = 0; i < 10; i++) { 2018 FileStatistics stat; 2019 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat)); 2020 EXPECT_EQ(1010, stat.length); 2021 EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1); 2022 EXPECT_FALSE(stat.is_directory); 2023 TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder", &stat)); 2024 EXPECT_EQ(0, stat.length); 2025 EXPECT_EQ(0, stat.mtime_nsec); 2026 EXPECT_TRUE(stat.is_directory); 2027 } 2028 } 2029 2030 TEST(GcsFileSystemTest, Stat_Cache_Flush) { 2031 std::vector<HttpRequest*> requests( 2032 {new FakeHttpRequest( 2033 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 2034 "file.txt?fields=size%2Cupdated\n" 2035 "Auth Token: fake_token\n" 2036 "Timeouts: 5 1 10\n", 2037 strings::StrCat("{\"size\": \"1010\"," 2038 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")), 2039 new FakeHttpRequest( 2040 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 2041 "file.txt?fields=size%2Cupdated\n" 2042 "Auth Token: fake_token\n" 2043 "Timeouts: 5 1 10\n", 2044 strings::StrCat("{\"size\": \"1010\"," 2045 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); 2046 GcsFileSystem fs( 2047 std::unique_ptr<AuthProvider>(new FakeAuthProvider), 2048 std::unique_ptr<HttpRequest::Factory>( 2049 new FakeHttpRequestFactory(&requests)), 2050 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 2051 3600 /* stat cache max age */, 0 /* stat cache max entries */, 2052 0 /* matching paths cache max age */, 2053 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, 2054 kTestTimeoutConfig, nullptr /* gcs additional header */); 2055 // There should be a single HTTP request to GCS for fs.Stat in this loop. 2056 for (int i = 0; i < 10; i++) { 2057 FileStatistics stat; 2058 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat)); 2059 EXPECT_EQ(1010, stat.length); 2060 EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1); 2061 EXPECT_FALSE(stat.is_directory); 2062 } 2063 // After flushing caches, there should be a second request to GCS for fs.Stat. 2064 fs.FlushCaches(); 2065 for (int i = 0; i < 10; i++) { 2066 FileStatistics stat; 2067 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat)); 2068 EXPECT_EQ(1010, stat.length); 2069 EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1); 2070 EXPECT_FALSE(stat.is_directory); 2071 } 2072 } 2073 2074 TEST(GcsFileSystemTest, IsDirectory_NotFound) { 2075 std::vector<HttpRequest*> requests( 2076 {new FakeHttpRequest( 2077 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 2078 "fields=items%2Fname%2CnextPageToken&prefix=file.txt%2F" 2079 "&maxResults=1\n" 2080 "Auth Token: fake_token\n" 2081 "Timeouts: 5 1 10\n", 2082 "{}"), 2083 new FakeHttpRequest( 2084 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 2085 "file.txt?fields=size%2Cupdated\n" 2086 "Auth Token: fake_token\n" 2087 "Timeouts: 5 1 10\n", 2088 "", errors::NotFound("404"), 404)}); 2089 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 2090 std::unique_ptr<HttpRequest::Factory>( 2091 new FakeHttpRequestFactory(&requests)), 2092 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 2093 0 /* stat cache max age */, 0 /* stat cache max entries */, 2094 0 /* matching paths cache max age */, 2095 0 /* matching paths cache max entries */, 2096 0 /* initial retry delay*/, kTestTimeoutConfig, 2097 nullptr /* gcs additional header */); 2098 2099 EXPECT_EQ(error::Code::NOT_FOUND, 2100 fs.IsDirectory("gs://bucket/file.txt").code()); 2101 } 2102 2103 TEST(GcsFileSystemTest, IsDirectory_NotDirectoryButObject) { 2104 std::vector<HttpRequest*> requests( 2105 {new FakeHttpRequest( 2106 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 2107 "fields=items%2Fname%2CnextPageToken&prefix=file.txt%2F" 2108 "&maxResults=1\n" 2109 "Auth Token: fake_token\n" 2110 "Timeouts: 5 1 10\n", 2111 "{}"), 2112 new FakeHttpRequest( 2113 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 2114 "file.txt?fields=size%2Cupdated\n" 2115 "Auth Token: fake_token\n" 2116 "Timeouts: 5 1 10\n", 2117 strings::StrCat("{\"size\": \"1010\"," 2118 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); 2119 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 2120 std::unique_ptr<HttpRequest::Factory>( 2121 new FakeHttpRequestFactory(&requests)), 2122 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 2123 0 /* stat cache max age */, 0 /* stat cache max entries */, 2124 0 /* matching paths cache max age */, 2125 0 /* matching paths cache max entries */, 2126 0 /* initial retry delay*/, kTestTimeoutConfig, 2127 nullptr /* gcs additional header */); 2128 2129 EXPECT_EQ(error::Code::FAILED_PRECONDITION, 2130 fs.IsDirectory("gs://bucket/file.txt").code()); 2131 } 2132 2133 TEST(GcsFileSystemTest, IsDirectory_Yes) { 2134 std::vector<HttpRequest*> requests( 2135 {new FakeHttpRequest( 2136 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 2137 "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F" 2138 "&maxResults=1\n" 2139 "Auth Token: fake_token\n" 2140 "Timeouts: 5 1 10\n", 2141 "{\"items\": [{\"name\": \"subfolder/\"}]}"), 2142 new FakeHttpRequest( 2143 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 2144 "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F" 2145 "&maxResults=1\n" 2146 "Auth Token: fake_token\n" 2147 "Timeouts: 5 1 10\n", 2148 "{\"items\": [{\"name\": \"subfolder/\"}]}")}); 2149 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 2150 std::unique_ptr<HttpRequest::Factory>( 2151 new FakeHttpRequestFactory(&requests)), 2152 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 2153 0 /* stat cache max age */, 0 /* stat cache max entries */, 2154 0 /* matching paths cache max age */, 2155 0 /* matching paths cache max entries */, 2156 0 /* initial retry delay*/, kTestTimeoutConfig, 2157 nullptr /* gcs additional header */); 2158 2159 TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder")); 2160 TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder/")); 2161 } 2162 2163 TEST(GcsFileSystemTest, IsDirectory_Bucket) { 2164 std::vector<HttpRequest*> requests( 2165 {new FakeHttpRequest( 2166 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n" 2167 "Auth Token: fake_token\n" 2168 "Timeouts: 5 1 10\n", 2169 "{}"), 2170 new FakeHttpRequest( 2171 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n" 2172 "Auth Token: fake_token\n" 2173 "Timeouts: 5 1 10\n", 2174 "{}")}); 2175 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 2176 std::unique_ptr<HttpRequest::Factory>( 2177 new FakeHttpRequestFactory(&requests)), 2178 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 2179 0 /* stat cache max age */, 0 /* stat cache max entries */, 2180 0 /* matching paths cache max age */, 2181 0 /* matching paths cache max entries */, 2182 0 /* initial retry delay*/, kTestTimeoutConfig, 2183 nullptr /* gcs additional header */); 2184 2185 TF_EXPECT_OK(fs.IsDirectory("gs://bucket")); 2186 TF_EXPECT_OK(fs.IsDirectory("gs://bucket/")); 2187 } 2188 2189 TEST(GcsFileSystemTest, IsDirectory_BucketNotFound) { 2190 std::vector<HttpRequest*> requests({new FakeHttpRequest( 2191 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n" 2192 "Auth Token: fake_token\n" 2193 "Timeouts: 5 1 10\n", 2194 "", errors::NotFound("404"), 404)}); 2195 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 2196 std::unique_ptr<HttpRequest::Factory>( 2197 new FakeHttpRequestFactory(&requests)), 2198 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 2199 0 /* stat cache max age */, 0 /* stat cache max entries */, 2200 0 /* matching paths cache max age */, 2201 0 /* matching paths cache max entries */, 2202 0 /* initial retry delay*/, kTestTimeoutConfig, 2203 nullptr /* gcs additional header */); 2204 2205 EXPECT_EQ(error::Code::NOT_FOUND, fs.IsDirectory("gs://bucket/").code()); 2206 } 2207 2208 TEST(GcsFileSystemTest, CreateDir_Folder) { 2209 std::vector<HttpRequest*> requests( 2210 {new FakeHttpRequest( 2211 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?" 2212 "uploadType=resumable&name=subpath%2F\n" 2213 "Auth Token: fake_token\n" 2214 "Header X-Upload-Content-Length: 0\n" 2215 "Post: yes\n" 2216 "Timeouts: 5 1 10\n", 2217 "", {{"Location", "https://custom/upload/location"}}), 2218 new FakeHttpRequest("Uri: https://custom/upload/location\n" 2219 "Auth Token: fake_token\n" 2220 "Timeouts: 5 1 30\n" 2221 "Put body: \n", 2222 ""), 2223 new FakeHttpRequest( 2224 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?" 2225 "uploadType=resumable&name=subpath%2F\n" 2226 "Auth Token: fake_token\n" 2227 "Header X-Upload-Content-Length: 0\n" 2228 "Post: yes\n" 2229 "Timeouts: 5 1 10\n", 2230 "", {{"Location", "https://custom/upload/location"}}), 2231 new FakeHttpRequest("Uri: https://custom/upload/location\n" 2232 "Auth Token: fake_token\n" 2233 "Timeouts: 5 1 30\n" 2234 "Put body: \n", 2235 "")}); 2236 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 2237 std::unique_ptr<HttpRequest::Factory>( 2238 new FakeHttpRequestFactory(&requests)), 2239 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 2240 0 /* stat cache max age */, 0 /* stat cache max entries */, 2241 0 /* matching paths cache max age */, 2242 0 /* matching paths cache max entries */, 2243 0 /* initial retry delay*/, kTestTimeoutConfig, 2244 nullptr /* gcs additional header */); 2245 2246 TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath")); 2247 TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath/")); 2248 } 2249 2250 TEST(GcsFileSystemTest, CreateDir_Bucket) { 2251 std::vector<HttpRequest*> requests( 2252 {new FakeHttpRequest( 2253 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n" 2254 "Auth Token: fake_token\n" 2255 "Timeouts: 5 1 10\n", 2256 ""), 2257 new FakeHttpRequest( 2258 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n" 2259 "Auth Token: fake_token\n" 2260 "Timeouts: 5 1 10\n", 2261 "")}); 2262 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 2263 std::unique_ptr<HttpRequest::Factory>( 2264 new FakeHttpRequestFactory(&requests)), 2265 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 2266 0 /* stat cache max age */, 0 /* stat cache max entries */, 2267 0 /* matching paths cache max age */, 2268 0 /* matching paths cache max entries */, 2269 0 /* initial retry delay*/, kTestTimeoutConfig, 2270 nullptr /* gcs additional header */); 2271 2272 TF_EXPECT_OK(fs.CreateDir("gs://bucket/")); 2273 TF_EXPECT_OK(fs.CreateDir("gs://bucket")); 2274 } 2275 2276 TEST(GcsFileSystemTest, DeleteRecursively_Ok) { 2277 std::vector<HttpRequest*> requests( 2278 {// IsDirectory is checking whether there are children objects. 2279 new FakeHttpRequest( 2280 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 2281 "fields=items%2Fname%2CnextPageToken&prefix=path%2F" 2282 "&maxResults=1\n" 2283 "Auth Token: fake_token\n" 2284 "Timeouts: 5 1 10\n", 2285 "{\"items\": [ " 2286 " { \"name\": \"path/file1.txt\" }]}"), 2287 // GetChildren recursively. 2288 new FakeHttpRequest( 2289 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 2290 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n" 2291 "Auth Token: fake_token\n" 2292 "Timeouts: 5 1 10\n", 2293 "{\"items\": [ " 2294 " { \"name\": \"path/\" }," // The current directory's marker. 2295 " { \"name\": \"path/file1.txt\" }," 2296 " { \"name\": \"path/subpath/file2.txt\" }," 2297 " { \"name\": \"path/file3.txt\" }]}"), 2298 // Delete the current directory's marker. 2299 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" 2300 "/bucket/o/path%2F\n" 2301 "Auth Token: fake_token\n" 2302 "Timeouts: 5 1 10\n" 2303 "Delete: yes\n", 2304 ""), 2305 // Delete the object - fails and will be retried. 2306 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" 2307 "/bucket/o/path%2Ffile1.txt\n" 2308 "Auth Token: fake_token\n" 2309 "Timeouts: 5 1 10\n" 2310 "Delete: yes\n", 2311 "", errors::Unavailable("500"), 500), 2312 // Delete the object again. 2313 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" 2314 "/bucket/o/path%2Ffile1.txt\n" 2315 "Auth Token: fake_token\n" 2316 "Timeouts: 5 1 10\n" 2317 "Delete: yes\n", 2318 ""), 2319 // Delete the object. 2320 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" 2321 "/bucket/o/path%2Fsubpath%2Ffile2.txt\n" 2322 "Auth Token: fake_token\n" 2323 "Timeouts: 5 1 10\n" 2324 "Delete: yes\n", 2325 ""), 2326 // Delete the object. 2327 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" 2328 "/bucket/o/path%2Ffile3.txt\n" 2329 "Auth Token: fake_token\n" 2330 "Timeouts: 5 1 10\n" 2331 "Delete: yes\n", 2332 "")}); 2333 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 2334 std::unique_ptr<HttpRequest::Factory>( 2335 new FakeHttpRequestFactory(&requests)), 2336 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 2337 0 /* stat cache max age */, 0 /* stat cache max entries */, 2338 0 /* matching paths cache max age */, 2339 0 /* matching paths cache max entries */, 2340 0 /* initial retry delay*/, kTestTimeoutConfig, 2341 nullptr /* gcs additional header */); 2342 2343 int64 undeleted_files, undeleted_dirs; 2344 TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files, 2345 &undeleted_dirs)); 2346 EXPECT_EQ(0, undeleted_files); 2347 EXPECT_EQ(0, undeleted_dirs); 2348 } 2349 2350 TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) { 2351 std::vector<HttpRequest*> requests( 2352 {// IsDirectory is checking whether there are children objects. 2353 new FakeHttpRequest( 2354 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 2355 "fields=items%2Fname%2CnextPageToken&prefix=path%2F" 2356 "&maxResults=1\n" 2357 "Auth Token: fake_token\n" 2358 "Timeouts: 5 1 10\n", 2359 "{\"items\": [ " 2360 " { \"name\": \"path/file1.txt\" }]}"), 2361 // Calling GetChildren recursively. 2362 new FakeHttpRequest( 2363 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 2364 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n" 2365 "Auth Token: fake_token\n" 2366 "Timeouts: 5 1 10\n", 2367 "{\"items\": [ " 2368 " { \"name\": \"path/file1.txt\" }," 2369 " { \"name\": \"path/subpath/\" }," 2370 " { \"name\": \"path/subpath/file2.txt\" }," 2371 " { \"name\": \"path/file3.txt\" }]}"), 2372 // Deleting the object. 2373 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" 2374 "/bucket/o/path%2Ffile1.txt\n" 2375 "Auth Token: fake_token\n" 2376 "Timeouts: 5 1 10\n" 2377 "Delete: yes\n", 2378 ""), 2379 // Deleting the directory marker gs://bucket/path/ - fails with 404. 2380 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" 2381 "/bucket/o/path%2Fsubpath%2F\n" 2382 "Auth Token: fake_token\n" 2383 "Timeouts: 5 1 10\n" 2384 "Delete: yes\n", 2385 "", errors::NotFound("404"), 404), 2386 // Checking if gs://bucket/path/subpath/ is a folder - it is. 2387 new FakeHttpRequest( 2388 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 2389 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F" 2390 "&maxResults=1\n" 2391 "Auth Token: fake_token\n" 2392 "Timeouts: 5 1 10\n", 2393 strings::StrCat("{\"items\": [ " 2394 " { \"name\": \"path/subpath/\" }]}")), 2395 // Deleting the object gs://bucket/path/subpath/file2.txt 2396 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" 2397 "/bucket/o/path%2Fsubpath%2Ffile2.txt\n" 2398 "Auth Token: fake_token\n" 2399 "Timeouts: 5 1 10\n" 2400 "Delete: yes\n", 2401 ""), 2402 // Deleting the object s://bucket/path/file3.txt - fails with 404. 2403 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" 2404 "/bucket/o/path%2Ffile3.txt\n" 2405 "Auth Token: fake_token\n" 2406 "Timeouts: 5 1 10\n" 2407 "Delete: yes\n", 2408 "", errors::NotFound("404"), 404), 2409 // Checking if gs://bucket/path/file3.txt/ is a folder - it's not. 2410 new FakeHttpRequest( 2411 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 2412 "fields=items%2Fname%2CnextPageToken&prefix=path%2Ffile3.txt%2F" 2413 "&maxResults=1\n" 2414 "Auth Token: fake_token\n" 2415 "Timeouts: 5 1 10\n", 2416 "{}"), 2417 // Checking if gs://bucket/path/file3.txt is an object - fails with 404. 2418 new FakeHttpRequest( 2419 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 2420 "path%2Ffile3.txt?fields=size%2Cupdated\n" 2421 "Auth Token: fake_token\n" 2422 "Timeouts: 5 1 10\n", 2423 "", errors::NotFound("404"), 404)}); 2424 2425 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 2426 std::unique_ptr<HttpRequest::Factory>( 2427 new FakeHttpRequestFactory(&requests)), 2428 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 2429 0 /* stat cache max age */, 0 /* stat cache max entries */, 2430 0 /* matching paths cache max age */, 2431 0 /* matching paths cache max entries */, 2432 0 /* initial retry delay*/, kTestTimeoutConfig, 2433 nullptr /* gcs additional header */); 2434 2435 int64 undeleted_files, undeleted_dirs; 2436 TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files, 2437 &undeleted_dirs)); 2438 EXPECT_EQ(1, undeleted_files); 2439 EXPECT_EQ(1, undeleted_dirs); 2440 } 2441 2442 TEST(GcsFileSystemTest, DeleteRecursively_NotAFolder) { 2443 std::vector<HttpRequest*> requests( 2444 {// IsDirectory is checking whether there are children objects. 2445 new FakeHttpRequest( 2446 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" 2447 "fields=items%2Fname%2CnextPageToken&prefix=path%2F" 2448 "&maxResults=1\n" 2449 "Auth Token: fake_token\n" 2450 "Timeouts: 5 1 10\n", 2451 "{}"), 2452 // IsDirectory is checking if the path exists as an object. 2453 new FakeHttpRequest( 2454 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" 2455 "path?fields=size%2Cupdated\n" 2456 "Auth Token: fake_token\n" 2457 "Timeouts: 5 1 10\n", 2458 "", errors::NotFound("404"), 404)}); 2459 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 2460 std::unique_ptr<HttpRequest::Factory>( 2461 new FakeHttpRequestFactory(&requests)), 2462 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 2463 0 /* stat cache max age */, 0 /* stat cache max entries */, 2464 0 /* matching paths cache max age */, 2465 0 /* matching paths cache max entries */, 2466 0 /* initial retry delay*/, kTestTimeoutConfig, 2467 nullptr /* gcs additional header */); 2468 2469 int64 undeleted_files, undeleted_dirs; 2470 EXPECT_EQ(error::Code::NOT_FOUND, 2471 fs.DeleteRecursively("gs://bucket/path", &undeleted_files, 2472 &undeleted_dirs) 2473 .code()); 2474 EXPECT_EQ(0, undeleted_files); 2475 EXPECT_EQ(1, undeleted_dirs); 2476 } 2477 2478 TEST(GcsFileSystemTest, AdditionalRequestHeaderTest) { 2479 GcsFileSystem fs1; 2480 EXPECT_EQ("", fs1.additional_header_name()); 2481 EXPECT_EQ("", fs1.additional_header_value()); 2482 2483 setenv("GCS_ADDITIONAL_REQUEST_HEADER", 2484 "X-Add-Header:My Additional Header Value", 1); 2485 GcsFileSystem fs2; 2486 EXPECT_EQ("X-Add-Header", fs2.additional_header_name()); 2487 EXPECT_EQ("My Additional Header Value", fs2.additional_header_value()); 2488 2489 setenv("GCS_ADDITIONAL_REQUEST_HEADER", "Someinvalidheadervalue", 1); 2490 GcsFileSystem fs3; 2491 EXPECT_EQ("", fs3.additional_header_name()); 2492 EXPECT_EQ("", fs3.additional_header_value()); 2493 2494 setenv("GCS_ADDITIONAL_REQUEST_HEADER", ":thisisinvalid", 1); 2495 GcsFileSystem fs4; 2496 EXPECT_EQ("", fs4.additional_header_name()); 2497 EXPECT_EQ("", fs4.additional_header_value()); 2498 2499 setenv("GCS_ADDITIONAL_REQUEST_HEADER", "soisthis:", 1); 2500 GcsFileSystem fs5; 2501 EXPECT_EQ("", fs5.additional_header_name()); 2502 EXPECT_EQ("", fs5.additional_header_value()); 2503 2504 setenv("GCS_ADDITIONAL_REQUEST_HEADER", "a:b", 1); 2505 GcsFileSystem fs6; 2506 EXPECT_EQ("a", fs6.additional_header_name()); 2507 EXPECT_EQ("b", fs6.additional_header_value()); 2508 2509 auto* add_header = new std::pair<const string, const string>( 2510 "mynewheader", "newheadercontents"); 2511 2512 std::vector<HttpRequest*> requests( 2513 {// IsDirectory is checking whether there are children objects. 2514 new FakeHttpRequest("Uri: https://www.googleapis.com/fake\n" 2515 "Auth Token: fake_token\n" 2516 "Header mynewheader: newheadercontents\n" 2517 "Header Hello: world\n", 2518 "{}")}); 2519 GcsFileSystem fs7( 2520 std::unique_ptr<AuthProvider>(new FakeAuthProvider), 2521 std::unique_ptr<HttpRequest::Factory>( 2522 new FakeHttpRequestFactory(&requests)), 2523 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 2524 0 /* stat cache max age */, 0 /* stat cache max entries */, 2525 0 /* matching paths cache max age */, 2526 0 /* matching paths cache max entries */, 0 /* initial retry delay */, 2527 kTestTimeoutConfig, add_header /* gcs additional header */); 2528 2529 std::unique_ptr<HttpRequest> request; 2530 TF_EXPECT_OK(fs7.CreateHttpRequest(&request)); 2531 request->SetUri("https://www.googleapis.com/fake"); 2532 request->AddHeader("Hello", "world"); 2533 TF_EXPECT_OK(request->Send()); 2534 } 2535 2536 TEST(GcsFileSystemTest, OverrideCacheParameters) { 2537 // Verify defaults are propagated correctly. 2538 GcsFileSystem fs1; 2539 EXPECT_EQ(128 * 1024 * 1024, fs1.block_size()); 2540 EXPECT_EQ(2 * fs1.block_size(), fs1.max_bytes()); 2541 EXPECT_EQ(0, fs1.max_staleness()); 2542 EXPECT_EQ(120, fs1.timeouts().connect); 2543 EXPECT_EQ(60, fs1.timeouts().idle); 2544 EXPECT_EQ(3600, fs1.timeouts().metadata); 2545 EXPECT_EQ(3600, fs1.timeouts().read); 2546 EXPECT_EQ(3600, fs1.timeouts().write); 2547 2548 // Verify legacy readahead buffer override sets block size. 2549 setenv("GCS_READAHEAD_BUFFER_SIZE_BYTES", "123456789", 1); 2550 GcsFileSystem fs2; 2551 EXPECT_EQ(123456789L, fs2.block_size()); 2552 2553 // Verify block size, max size, and max staleness overrides. 2554 setenv("GCS_READ_CACHE_BLOCK_SIZE_MB", "1", 1); 2555 setenv("GCS_READ_CACHE_MAX_SIZE_MB", "16", 1); 2556 setenv("GCS_READ_CACHE_MAX_STALENESS", "60", 1); 2557 GcsFileSystem fs3; 2558 EXPECT_EQ(1048576L, fs3.block_size()); 2559 EXPECT_EQ(16 * 1024 * 1024, fs3.max_bytes()); 2560 EXPECT_EQ(60, fs3.max_staleness()); 2561 2562 // Verify StatCache and MatchingPathsCache overrides. 2563 setenv("GCS_STAT_CACHE_MAX_AGE", "60", 1); 2564 setenv("GCS_STAT_CACHE_MAX_ENTRIES", "32", 1); 2565 setenv("GCS_MATCHING_PATHS_CACHE_MAX_AGE", "30", 1); 2566 setenv("GCS_MATCHING_PATHS_CACHE_MAX_ENTRIES", "64", 1); 2567 GcsFileSystem fs4; 2568 EXPECT_EQ(60, fs4.stat_cache_max_age()); 2569 EXPECT_EQ(32, fs4.stat_cache_max_entries()); 2570 EXPECT_EQ(30, fs4.matching_paths_cache_max_age()); 2571 EXPECT_EQ(64, fs4.matching_paths_cache_max_entries()); 2572 2573 // Verify timeout overrides. 2574 setenv("GCS_REQUEST_CONNECTION_TIMEOUT_SECS", "10", 1); 2575 setenv("GCS_REQUEST_IDLE_TIMEOUT_SECS", "5", 1); 2576 setenv("GCS_METADATA_REQUEST_TIMEOUT_SECS", "20", 1); 2577 setenv("GCS_READ_REQUEST_TIMEOUT_SECS", "30", 1); 2578 setenv("GCS_WRITE_REQUEST_TIMEOUT_SECS", "40", 1); 2579 GcsFileSystem fs5; 2580 EXPECT_EQ(10, fs5.timeouts().connect); 2581 EXPECT_EQ(5, fs5.timeouts().idle); 2582 EXPECT_EQ(20, fs5.timeouts().metadata); 2583 EXPECT_EQ(30, fs5.timeouts().read); 2584 EXPECT_EQ(40, fs5.timeouts().write); 2585 } 2586 2587 TEST(GcsFileSystemTest, CreateHttpRequest) { 2588 std::vector<HttpRequest*> requests( 2589 {// IsDirectory is checking whether there are children objects. 2590 new FakeHttpRequest("Uri: https://www.googleapis.com/fake\n" 2591 "Auth Token: fake_token\n" 2592 "Header Hello: world\n", 2593 "{}")}); 2594 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider), 2595 std::unique_ptr<HttpRequest::Factory>( 2596 new FakeHttpRequestFactory(&requests)), 2597 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 2598 0 /* stat cache max age */, 0 /* stat cache max entries */, 2599 0 /* matching paths cache max age */, 2600 0 /* matching paths cache max entries */, 2601 0 /* initial retry delay */, kTestTimeoutConfig, 2602 nullptr /* gcs additional header */); 2603 2604 std::unique_ptr<HttpRequest> request; 2605 TF_EXPECT_OK(fs.CreateHttpRequest(&request)); 2606 request->SetUri("https://www.googleapis.com/fake"); 2607 request->AddHeader("Hello", "world"); 2608 TF_EXPECT_OK(request->Send()); 2609 } 2610 2611 } // namespace 2612 } // namespace tensorflow 2613
