1 /* 2 * Copyright (C) 2007 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include <dirent.h> 18 #include <inttypes.h> 19 #include <limits.h> 20 #include <stdio.h> 21 #include <stdlib.h> 22 #include <sys/stat.h> 23 #include <sys/time.h> 24 #include <sys/types.h> 25 #include <time.h> 26 #include <unistd.h> 27 #include <utime.h> 28 29 #include <chrono> 30 #include <functional> 31 #include <memory> 32 #include <sstream> 33 #include <string> 34 #include <vector> 35 36 #include "sysdeps.h" 37 38 #include "adb.h" 39 #include "adb_client.h" 40 #include "adb_io.h" 41 #include "adb_utils.h" 42 #include "file_sync_service.h" 43 #include "line_printer.h" 44 #include "sysdeps/errno.h" 45 #include "sysdeps/stat.h" 46 47 #include <android-base/file.h> 48 #include <android-base/strings.h> 49 #include <android-base/stringprintf.h> 50 51 struct syncsendbuf { 52 unsigned id; 53 unsigned size; 54 char data[SYNC_DATA_MAX]; 55 }; 56 57 static void ensure_trailing_separators(std::string& local_path, std::string& remote_path) { 58 if (!adb_is_separator(local_path.back())) { 59 local_path.push_back(OS_PATH_SEPARATOR); 60 } 61 if (remote_path.back() != '/') { 62 remote_path.push_back('/'); 63 } 64 } 65 66 static bool should_pull_file(mode_t mode) { 67 return S_ISREG(mode) || S_ISBLK(mode) || S_ISCHR(mode); 68 } 69 70 static bool should_push_file(mode_t mode) { 71 return S_ISREG(mode) || S_ISLNK(mode); 72 } 73 74 struct copyinfo { 75 std::string lpath; 76 std::string rpath; 77 int64_t time = 0; 78 uint32_t mode; 79 uint64_t size = 0; 80 bool skip = false; 81 82 copyinfo(const std::string& local_path, 83 const std::string& remote_path, 84 const std::string& name, 85 unsigned int mode) 86 : lpath(local_path), rpath(remote_path), mode(mode) { 87 ensure_trailing_separators(lpath, rpath); 88 lpath.append(name); 89 rpath.append(name); 90 if (S_ISDIR(mode)) { 91 ensure_trailing_separators(lpath, rpath); 92 } 93 } 94 }; 95 96 enum class TransferDirection { 97 push, 98 pull, 99 }; 100 101 struct TransferLedger { 102 std::chrono::steady_clock::time_point start_time; 103 uint64_t files_transferred; 104 uint64_t files_skipped; 105 uint64_t bytes_transferred; 106 uint64_t bytes_expected; 107 bool expect_multiple_files; 108 109 TransferLedger() { 110 Reset(); 111 } 112 113 bool operator==(const TransferLedger& other) const { 114 return files_transferred == other.files_transferred && 115 files_skipped == other.files_skipped && bytes_transferred == other.bytes_transferred; 116 } 117 118 bool operator!=(const TransferLedger& other) const { 119 return !(*this == other); 120 } 121 122 void Reset() { 123 start_time = std::chrono::steady_clock::now(); 124 files_transferred = 0; 125 files_skipped = 0; 126 bytes_transferred = 0; 127 bytes_expected = 0; 128 } 129 130 std::string TransferRate() { 131 if (bytes_transferred == 0) return ""; 132 133 std::chrono::duration<double> duration; 134 duration = std::chrono::steady_clock::now() - start_time; 135 136 double s = duration.count(); 137 if (s == 0) { 138 return ""; 139 } 140 double rate = (static_cast<double>(bytes_transferred) / s) / (1024 * 1024); 141 return android::base::StringPrintf(" %.1f MB/s (%" PRIu64 " bytes in %.3fs)", rate, 142 bytes_transferred, s); 143 } 144 145 void ReportProgress(LinePrinter& lp, const std::string& file, uint64_t file_copied_bytes, 146 uint64_t file_total_bytes) { 147 char overall_percentage_str[5] = "?"; 148 if (bytes_expected != 0 && bytes_transferred <= bytes_expected) { 149 int overall_percentage = static_cast<int>(bytes_transferred * 100 / bytes_expected); 150 // If we're pulling symbolic links, we'll pull the target of the link rather than 151 // just create a local link, and that will cause us to go over 100%. 152 if (overall_percentage <= 100) { 153 snprintf(overall_percentage_str, sizeof(overall_percentage_str), "%d%%", 154 overall_percentage); 155 } 156 } 157 158 std::string output; 159 if (file_copied_bytes > file_total_bytes || file_total_bytes == 0) { 160 // This case can happen if we're racing against something that wrote to the file 161 // between our stat and our read, or if we're reading a magic file that lies about 162 // its size. Just show how much we've copied. 163 output = android::base::StringPrintf("[%4s] %s: %" PRId64 "/?", overall_percentage_str, 164 file.c_str(), file_copied_bytes); 165 } else { 166 // If we're transferring multiple files, we want to know how far through the current 167 // file we are, as well as the overall percentage. 168 if (expect_multiple_files) { 169 int file_percentage = static_cast<int>(file_copied_bytes * 100 / file_total_bytes); 170 output = android::base::StringPrintf("[%4s] %s: %d%%", overall_percentage_str, 171 file.c_str(), file_percentage); 172 } else { 173 output = 174 android::base::StringPrintf("[%4s] %s", overall_percentage_str, file.c_str()); 175 } 176 } 177 lp.Print(output, LinePrinter::LineType::INFO); 178 } 179 180 void ReportTransferRate(LinePrinter& lp, const std::string& name, TransferDirection direction) { 181 const char* direction_str = (direction == TransferDirection::push) ? "pushed" : "pulled"; 182 std::stringstream ss; 183 if (!name.empty()) { 184 ss << name << ": "; 185 } 186 ss << files_transferred << " file" << ((files_transferred == 1) ? "" : "s") << " " 187 << direction_str << "."; 188 if (files_skipped > 0) { 189 ss << " " << files_skipped << " file" << ((files_skipped == 1) ? "" : "s") 190 << " skipped."; 191 } 192 ss << TransferRate(); 193 194 lp.Print(ss.str(), LinePrinter::LineType::INFO); 195 lp.KeepInfoLine(); 196 } 197 }; 198 199 class SyncConnection { 200 public: 201 SyncConnection() : expect_done_(false) { 202 max = SYNC_DATA_MAX; // TODO: decide at runtime. 203 204 std::string error; 205 FeatureSet features; 206 if (!adb_get_feature_set(&features, &error)) { 207 fd = -1; 208 Error("failed to get feature set: %s", error.c_str()); 209 } else { 210 have_stat_v2_ = CanUseFeature(features, kFeatureStat2); 211 fd = adb_connect("sync:", &error); 212 if (fd < 0) { 213 Error("connect failed: %s", error.c_str()); 214 } 215 } 216 } 217 218 ~SyncConnection() { 219 if (!IsValid()) return; 220 221 if (SendQuit()) { 222 // We sent a quit command, so the server should be doing orderly 223 // shutdown soon. But if we encountered an error while we were using 224 // the connection, the server might still be sending data (before 225 // doing orderly shutdown), in which case we won't wait for all of 226 // the data nor the coming orderly shutdown. In the common success 227 // case, this will wait for the server to do orderly shutdown. 228 ReadOrderlyShutdown(fd); 229 } 230 adb_close(fd); 231 232 line_printer_.KeepInfoLine(); 233 } 234 235 bool IsValid() { return fd >= 0; } 236 237 bool ReceivedError(const char* from, const char* to) { 238 adb_pollfd pfd = {.fd = fd, .events = POLLIN}; 239 int rc = adb_poll(&pfd, 1, 0); 240 if (rc < 0) { 241 Error("failed to poll: %s", strerror(errno)); 242 return true; 243 } 244 return rc != 0; 245 } 246 247 void NewTransfer() { 248 current_ledger_.Reset(); 249 } 250 251 void RecordBytesTransferred(size_t bytes) { 252 current_ledger_.bytes_transferred += bytes; 253 global_ledger_.bytes_transferred += bytes; 254 } 255 256 void RecordFilesTransferred(size_t files) { 257 current_ledger_.files_transferred += files; 258 global_ledger_.files_transferred += files; 259 } 260 261 void RecordFilesSkipped(size_t files) { 262 current_ledger_.files_skipped += files; 263 global_ledger_.files_skipped += files; 264 } 265 266 void ReportProgress(const std::string& file, uint64_t file_copied_bytes, 267 uint64_t file_total_bytes) { 268 current_ledger_.ReportProgress(line_printer_, file, file_copied_bytes, file_total_bytes); 269 } 270 271 void ReportTransferRate(const std::string& file, TransferDirection direction) { 272 current_ledger_.ReportTransferRate(line_printer_, file, direction); 273 } 274 275 void ReportOverallTransferRate(TransferDirection direction) { 276 if (current_ledger_ != global_ledger_) { 277 global_ledger_.ReportTransferRate(line_printer_, "", direction); 278 } 279 } 280 281 bool SendRequest(int id, const char* path_and_mode) { 282 size_t path_length = strlen(path_and_mode); 283 if (path_length > 1024) { 284 Error("SendRequest failed: path too long: %zu", path_length); 285 errno = ENAMETOOLONG; 286 return false; 287 } 288 289 // Sending header and payload in a single write makes a noticeable 290 // difference to "adb sync" performance. 291 std::vector<char> buf(sizeof(SyncRequest) + path_length); 292 SyncRequest* req = reinterpret_cast<SyncRequest*>(&buf[0]); 293 req->id = id; 294 req->path_length = path_length; 295 char* data = reinterpret_cast<char*>(req + 1); 296 memcpy(data, path_and_mode, path_length); 297 298 return WriteFdExactly(fd, &buf[0], buf.size()); 299 } 300 301 bool SendStat(const char* path_and_mode) { 302 if (!have_stat_v2_) { 303 errno = ENOTSUP; 304 return false; 305 } 306 return SendRequest(ID_STAT_V2, path_and_mode); 307 } 308 309 bool SendLstat(const char* path_and_mode) { 310 if (have_stat_v2_) { 311 return SendRequest(ID_LSTAT_V2, path_and_mode); 312 } else { 313 return SendRequest(ID_LSTAT_V1, path_and_mode); 314 } 315 } 316 317 bool FinishStat(struct stat* st) { 318 syncmsg msg; 319 320 memset(st, 0, sizeof(*st)); 321 if (have_stat_v2_) { 322 if (!ReadFdExactly(fd, &msg.stat_v2, sizeof(msg.stat_v2))) { 323 fatal_errno("protocol fault: failed to read stat response"); 324 } 325 326 if (msg.stat_v2.id != ID_LSTAT_V2 && msg.stat_v2.id != ID_STAT_V2) { 327 fatal_errno("protocol fault: stat response has wrong message id: %" PRIx32, 328 msg.stat_v2.id); 329 } 330 331 if (msg.stat_v2.error != 0) { 332 errno = errno_from_wire(msg.stat_v2.error); 333 return false; 334 } 335 336 st->st_dev = msg.stat_v2.dev; 337 st->st_ino = msg.stat_v2.ino; 338 st->st_mode = msg.stat_v2.mode; 339 st->st_nlink = msg.stat_v2.nlink; 340 st->st_uid = msg.stat_v2.uid; 341 st->st_gid = msg.stat_v2.gid; 342 st->st_size = msg.stat_v2.size; 343 st->st_atime = msg.stat_v2.atime; 344 st->st_mtime = msg.stat_v2.mtime; 345 st->st_ctime = msg.stat_v2.ctime; 346 return true; 347 } else { 348 if (!ReadFdExactly(fd, &msg.stat_v1, sizeof(msg.stat_v1))) { 349 fatal_errno("protocol fault: failed to read stat response"); 350 } 351 352 if (msg.stat_v1.id != ID_LSTAT_V1) { 353 fatal_errno("protocol fault: stat response has wrong message id: %" PRIx32, 354 msg.stat_v1.id); 355 } 356 357 if (msg.stat_v1.mode == 0 && msg.stat_v1.size == 0 && msg.stat_v1.time == 0) { 358 // There's no way for us to know what the error was. 359 errno = ENOPROTOOPT; 360 return false; 361 } 362 363 st->st_mode = msg.stat_v1.mode; 364 st->st_size = msg.stat_v1.size; 365 st->st_ctime = msg.stat_v1.time; 366 st->st_mtime = msg.stat_v1.time; 367 } 368 369 return true; 370 } 371 372 // Sending header, payload, and footer in a single write makes a huge 373 // difference to "adb sync" performance. 374 bool SendSmallFile(const char* path_and_mode, 375 const char* lpath, const char* rpath, 376 unsigned mtime, 377 const char* data, size_t data_length) { 378 size_t path_length = strlen(path_and_mode); 379 if (path_length > 1024) { 380 Error("SendSmallFile failed: path too long: %zu", path_length); 381 errno = ENAMETOOLONG; 382 return false; 383 } 384 385 std::vector<char> buf(sizeof(SyncRequest) + path_length + 386 sizeof(SyncRequest) + data_length + 387 sizeof(SyncRequest)); 388 char* p = &buf[0]; 389 390 SyncRequest* req_send = reinterpret_cast<SyncRequest*>(p); 391 req_send->id = ID_SEND; 392 req_send->path_length = path_length; 393 p += sizeof(SyncRequest); 394 memcpy(p, path_and_mode, path_length); 395 p += path_length; 396 397 SyncRequest* req_data = reinterpret_cast<SyncRequest*>(p); 398 req_data->id = ID_DATA; 399 req_data->path_length = data_length; 400 p += sizeof(SyncRequest); 401 memcpy(p, data, data_length); 402 p += data_length; 403 404 SyncRequest* req_done = reinterpret_cast<SyncRequest*>(p); 405 req_done->id = ID_DONE; 406 req_done->path_length = mtime; 407 p += sizeof(SyncRequest); 408 409 WriteOrDie(lpath, rpath, &buf[0], (p - &buf[0])); 410 expect_done_ = true; 411 412 // RecordFilesTransferred gets called in CopyDone. 413 RecordBytesTransferred(data_length); 414 ReportProgress(rpath, data_length, data_length); 415 return true; 416 } 417 418 bool SendLargeFile(const char* path_and_mode, 419 const char* lpath, const char* rpath, 420 unsigned mtime) { 421 if (!SendRequest(ID_SEND, path_and_mode)) { 422 Error("failed to send ID_SEND message '%s': %s", path_and_mode, strerror(errno)); 423 return false; 424 } 425 426 struct stat st; 427 if (stat(lpath, &st) == -1) { 428 Error("cannot stat '%s': %s", lpath, strerror(errno)); 429 return false; 430 } 431 432 uint64_t total_size = st.st_size; 433 uint64_t bytes_copied = 0; 434 435 int lfd = adb_open(lpath, O_RDONLY); 436 if (lfd < 0) { 437 Error("opening '%s' locally failed: %s", lpath, strerror(errno)); 438 return false; 439 } 440 441 syncsendbuf sbuf; 442 sbuf.id = ID_DATA; 443 while (true) { 444 int bytes_read = adb_read(lfd, sbuf.data, max); 445 if (bytes_read == -1) { 446 Error("reading '%s' locally failed: %s", lpath, strerror(errno)); 447 adb_close(lfd); 448 return false; 449 } else if (bytes_read == 0) { 450 break; 451 } 452 453 sbuf.size = bytes_read; 454 WriteOrDie(lpath, rpath, &sbuf, sizeof(SyncRequest) + bytes_read); 455 456 RecordBytesTransferred(bytes_read); 457 bytes_copied += bytes_read; 458 459 // Check to see if we've received an error from the other side. 460 if (ReceivedError(lpath, rpath)) { 461 break; 462 } 463 464 ReportProgress(rpath, bytes_copied, total_size); 465 } 466 467 adb_close(lfd); 468 469 syncmsg msg; 470 msg.data.id = ID_DONE; 471 msg.data.size = mtime; 472 expect_done_ = true; 473 474 // RecordFilesTransferred gets called in CopyDone. 475 return WriteOrDie(lpath, rpath, &msg.data, sizeof(msg.data)); 476 } 477 478 bool CopyDone(const char* from, const char* to) { 479 syncmsg msg; 480 if (!ReadFdExactly(fd, &msg.status, sizeof(msg.status))) { 481 Error("failed to copy '%s' to '%s': couldn't read from device", from, to); 482 return false; 483 } 484 if (msg.status.id == ID_OKAY) { 485 if (expect_done_) { 486 expect_done_ = false; 487 RecordFilesTransferred(1); 488 return true; 489 } else { 490 Error("failed to copy '%s' to '%s': received premature success", from, to); 491 return true; 492 } 493 } 494 if (msg.status.id != ID_FAIL) { 495 Error("failed to copy '%s' to '%s': unknown reason %d", from, to, msg.status.id); 496 return false; 497 } 498 return ReportCopyFailure(from, to, msg); 499 } 500 501 bool ReportCopyFailure(const char* from, const char* to, const syncmsg& msg) { 502 std::vector<char> buf(msg.status.msglen + 1); 503 if (!ReadFdExactly(fd, &buf[0], msg.status.msglen)) { 504 Error("failed to copy '%s' to '%s'; failed to read reason (!): %s", 505 from, to, strerror(errno)); 506 return false; 507 } 508 buf[msg.status.msglen] = 0; 509 Error("failed to copy '%s' to '%s': remote %s", from, to, &buf[0]); 510 return false; 511 } 512 513 514 void Printf(const char* fmt, ...) __attribute__((__format__(ADB_FORMAT_ARCHETYPE, 2, 3))) { 515 std::string s; 516 517 va_list ap; 518 va_start(ap, fmt); 519 android::base::StringAppendV(&s, fmt, ap); 520 va_end(ap); 521 522 line_printer_.Print(s, LinePrinter::INFO); 523 } 524 525 void Println(const char* fmt, ...) __attribute__((__format__(ADB_FORMAT_ARCHETYPE, 2, 3))) { 526 std::string s; 527 528 va_list ap; 529 va_start(ap, fmt); 530 android::base::StringAppendV(&s, fmt, ap); 531 va_end(ap); 532 533 line_printer_.Print(s, LinePrinter::INFO); 534 line_printer_.KeepInfoLine(); 535 } 536 537 void Error(const char* fmt, ...) __attribute__((__format__(ADB_FORMAT_ARCHETYPE, 2, 3))) { 538 std::string s = "adb: error: "; 539 540 va_list ap; 541 va_start(ap, fmt); 542 android::base::StringAppendV(&s, fmt, ap); 543 va_end(ap); 544 545 line_printer_.Print(s, LinePrinter::ERROR); 546 } 547 548 void Warning(const char* fmt, ...) __attribute__((__format__(ADB_FORMAT_ARCHETYPE, 2, 3))) { 549 std::string s = "adb: warning: "; 550 551 va_list ap; 552 va_start(ap, fmt); 553 android::base::StringAppendV(&s, fmt, ap); 554 va_end(ap); 555 556 line_printer_.Print(s, LinePrinter::WARNING); 557 } 558 559 void ComputeExpectedTotalBytes(const std::vector<copyinfo>& file_list) { 560 current_ledger_.bytes_expected = 0; 561 for (const copyinfo& ci : file_list) { 562 // Unfortunately, this doesn't work for symbolic links, because we'll copy the 563 // target of the link rather than just creating a link. (But ci.size is the link size.) 564 if (!ci.skip) current_ledger_.bytes_expected += ci.size; 565 } 566 current_ledger_.expect_multiple_files = true; 567 } 568 569 void SetExpectedTotalBytes(uint64_t expected_total_bytes) { 570 current_ledger_.bytes_expected = expected_total_bytes; 571 current_ledger_.expect_multiple_files = false; 572 } 573 574 // TODO: add a char[max] buffer here, to replace syncsendbuf... 575 int fd; 576 size_t max; 577 578 private: 579 bool expect_done_; 580 bool have_stat_v2_; 581 582 TransferLedger global_ledger_; 583 TransferLedger current_ledger_; 584 LinePrinter line_printer_; 585 586 bool SendQuit() { 587 return SendRequest(ID_QUIT, ""); // TODO: add a SendResponse? 588 } 589 590 bool WriteOrDie(const char* from, const char* to, const void* data, size_t data_length) { 591 if (!WriteFdExactly(fd, data, data_length)) { 592 if (errno == ECONNRESET) { 593 // Assume adbd told us why it was closing the connection, and 594 // try to read failure reason from adbd. 595 syncmsg msg; 596 if (!ReadFdExactly(fd, &msg.status, sizeof(msg.status))) { 597 Error("failed to copy '%s' to '%s': no response: %s", from, to, strerror(errno)); 598 } else if (msg.status.id != ID_FAIL) { 599 Error("failed to copy '%s' to '%s': not ID_FAIL: %d", from, to, msg.status.id); 600 } else { 601 ReportCopyFailure(from, to, msg); 602 } 603 } else { 604 Error("%zu-byte write failed: %s", data_length, strerror(errno)); 605 } 606 _exit(1); 607 } 608 return true; 609 } 610 }; 611 612 typedef void (sync_ls_cb)(unsigned mode, unsigned size, unsigned time, const char* name); 613 614 static bool sync_ls(SyncConnection& sc, const char* path, 615 const std::function<sync_ls_cb>& func) { 616 if (!sc.SendRequest(ID_LIST, path)) return false; 617 618 while (true) { 619 syncmsg msg; 620 if (!ReadFdExactly(sc.fd, &msg.dent, sizeof(msg.dent))) return false; 621 622 if (msg.dent.id == ID_DONE) return true; 623 if (msg.dent.id != ID_DENT) return false; 624 625 size_t len = msg.dent.namelen; 626 if (len > 256) return false; // TODO: resize buffer? continue? 627 628 char buf[257]; 629 if (!ReadFdExactly(sc.fd, buf, len)) return false; 630 buf[len] = 0; 631 632 func(msg.dent.mode, msg.dent.size, msg.dent.time, buf); 633 } 634 } 635 636 static bool sync_stat(SyncConnection& sc, const char* path, struct stat* st) { 637 return sc.SendStat(path) && sc.FinishStat(st); 638 } 639 640 static bool sync_lstat(SyncConnection& sc, const char* path, struct stat* st) { 641 return sc.SendLstat(path) && sc.FinishStat(st); 642 } 643 644 static bool sync_stat_fallback(SyncConnection& sc, const char* path, struct stat* st) { 645 if (sync_stat(sc, path, st)) { 646 return true; 647 } 648 649 if (errno != ENOTSUP) { 650 return false; 651 } 652 653 // Try to emulate the parts we can when talking to older adbds. 654 bool lstat_result = sync_lstat(sc, path, st); 655 if (!lstat_result) { 656 return false; 657 } 658 659 if (S_ISLNK(st->st_mode)) { 660 // If the target is a symlink, figure out whether it's a file or a directory. 661 // Also, zero out the st_size field, since no one actually cares what the path length is. 662 st->st_size = 0; 663 std::string dir_path = path; 664 dir_path.push_back('/'); 665 struct stat tmp_st; 666 667 st->st_mode &= ~S_IFMT; 668 if (sync_lstat(sc, dir_path.c_str(), &tmp_st)) { 669 st->st_mode |= S_IFDIR; 670 } else { 671 st->st_mode |= S_IFREG; 672 } 673 } 674 return true; 675 } 676 677 static bool sync_send(SyncConnection& sc, const char* lpath, const char* rpath, 678 unsigned mtime, mode_t mode) 679 { 680 std::string path_and_mode = android::base::StringPrintf("%s,%d", rpath, mode); 681 682 if (S_ISLNK(mode)) { 683 #if !defined(_WIN32) 684 char buf[PATH_MAX]; 685 ssize_t data_length = readlink(lpath, buf, PATH_MAX - 1); 686 if (data_length == -1) { 687 sc.Error("readlink '%s' failed: %s", lpath, strerror(errno)); 688 return false; 689 } 690 buf[data_length++] = '\0'; 691 692 if (!sc.SendSmallFile(path_and_mode.c_str(), lpath, rpath, mtime, buf, data_length)) { 693 return false; 694 } 695 return sc.CopyDone(lpath, rpath); 696 #endif 697 } 698 699 struct stat st; 700 if (stat(lpath, &st) == -1) { 701 sc.Error("failed to stat local file '%s': %s", lpath, strerror(errno)); 702 return false; 703 } 704 if (st.st_size < SYNC_DATA_MAX) { 705 std::string data; 706 if (!android::base::ReadFileToString(lpath, &data, true)) { 707 sc.Error("failed to read all of '%s': %s", lpath, strerror(errno)); 708 return false; 709 } 710 if (!sc.SendSmallFile(path_and_mode.c_str(), lpath, rpath, mtime, 711 data.data(), data.size())) { 712 return false; 713 } 714 } else { 715 if (!sc.SendLargeFile(path_and_mode.c_str(), lpath, rpath, mtime)) { 716 return false; 717 } 718 } 719 return sc.CopyDone(lpath, rpath); 720 } 721 722 static bool sync_recv(SyncConnection& sc, const char* rpath, const char* lpath, 723 const char* name, uint64_t expected_size) { 724 if (!sc.SendRequest(ID_RECV, rpath)) return false; 725 726 adb_unlink(lpath); 727 int lfd = adb_creat(lpath, 0644); 728 if (lfd < 0) { 729 sc.Error("cannot create '%s': %s", lpath, strerror(errno)); 730 return false; 731 } 732 733 uint64_t bytes_copied = 0; 734 while (true) { 735 syncmsg msg; 736 if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) { 737 adb_close(lfd); 738 adb_unlink(lpath); 739 return false; 740 } 741 742 if (msg.data.id == ID_DONE) break; 743 744 if (msg.data.id != ID_DATA) { 745 adb_close(lfd); 746 adb_unlink(lpath); 747 sc.ReportCopyFailure(rpath, lpath, msg); 748 return false; 749 } 750 751 if (msg.data.size > sc.max) { 752 sc.Error("msg.data.size too large: %u (max %zu)", msg.data.size, sc.max); 753 adb_close(lfd); 754 adb_unlink(lpath); 755 return false; 756 } 757 758 char buffer[SYNC_DATA_MAX]; 759 if (!ReadFdExactly(sc.fd, buffer, msg.data.size)) { 760 adb_close(lfd); 761 adb_unlink(lpath); 762 return false; 763 } 764 765 if (!WriteFdExactly(lfd, buffer, msg.data.size)) { 766 sc.Error("cannot write '%s': %s", lpath, strerror(errno)); 767 adb_close(lfd); 768 adb_unlink(lpath); 769 return false; 770 } 771 772 bytes_copied += msg.data.size; 773 774 sc.RecordBytesTransferred(msg.data.size); 775 sc.ReportProgress(name != nullptr ? name : rpath, bytes_copied, expected_size); 776 } 777 778 sc.RecordFilesTransferred(1); 779 adb_close(lfd); 780 return true; 781 } 782 783 bool do_sync_ls(const char* path) { 784 SyncConnection sc; 785 if (!sc.IsValid()) return false; 786 787 return sync_ls(sc, path, [](unsigned mode, unsigned size, unsigned time, 788 const char* name) { 789 printf("%08x %08x %08x %s\n", mode, size, time, name); 790 }); 791 } 792 793 static bool IsDotOrDotDot(const char* name) { 794 return name[0] == '.' && (name[1] == '\0' || (name[1] == '.' && name[2] == '\0')); 795 } 796 797 static bool local_build_list(SyncConnection& sc, std::vector<copyinfo>* file_list, 798 const std::string& lpath, 799 const std::string& rpath) { 800 std::vector<copyinfo> dirlist; 801 std::unique_ptr<DIR, int (*)(DIR*)> dir(opendir(lpath.c_str()), closedir); 802 if (!dir) { 803 sc.Error("cannot open '%s': %s", lpath.c_str(), strerror(errno)); 804 return false; 805 } 806 807 bool empty_dir = true; 808 dirent* de; 809 while ((de = readdir(dir.get()))) { 810 if (IsDotOrDotDot(de->d_name)) { 811 continue; 812 } 813 814 empty_dir = false; 815 std::string stat_path = lpath + de->d_name; 816 817 struct stat st; 818 if (lstat(stat_path.c_str(), &st) == -1) { 819 sc.Error("cannot lstat '%s': %s", stat_path.c_str(), 820 strerror(errno)); 821 continue; 822 } 823 824 copyinfo ci(lpath, rpath, de->d_name, st.st_mode); 825 if (S_ISDIR(st.st_mode)) { 826 dirlist.push_back(ci); 827 } else { 828 if (!should_push_file(st.st_mode)) { 829 sc.Warning("skipping special file '%s' (mode = 0o%o)", lpath.c_str(), st.st_mode); 830 ci.skip = true; 831 } 832 ci.time = st.st_mtime; 833 ci.size = st.st_size; 834 file_list->push_back(ci); 835 } 836 } 837 838 // Close this directory and recurse. 839 dir.reset(); 840 841 // Add the current directory to the list if it was empty, to ensure that 842 // it gets created. 843 if (empty_dir) { 844 // TODO(b/25566053): Make pushing empty directories work. 845 // TODO(b/25457350): We don't preserve permissions on directories. 846 sc.Warning("skipping empty directory '%s'", lpath.c_str()); 847 copyinfo ci(android::base::Dirname(lpath), android::base::Dirname(rpath), 848 android::base::Basename(lpath), S_IFDIR); 849 ci.skip = true; 850 file_list->push_back(ci); 851 return true; 852 } 853 854 for (const copyinfo& ci : dirlist) { 855 local_build_list(sc, file_list, ci.lpath, ci.rpath); 856 } 857 858 return true; 859 } 860 861 static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, 862 std::string rpath, bool check_timestamps, 863 bool list_only) { 864 sc.NewTransfer(); 865 866 // Make sure that both directory paths end in a slash. 867 // Both paths are known to be nonempty, so we don't need to check. 868 ensure_trailing_separators(lpath, rpath); 869 870 // Recursively build the list of files to copy. 871 std::vector<copyinfo> file_list; 872 int skipped = 0; 873 if (!local_build_list(sc, &file_list, lpath, rpath)) { 874 return false; 875 } 876 877 if (check_timestamps) { 878 for (const copyinfo& ci : file_list) { 879 if (!sc.SendLstat(ci.rpath.c_str())) { 880 sc.Error("failed to send lstat"); 881 return false; 882 } 883 } 884 for (copyinfo& ci : file_list) { 885 struct stat st; 886 if (sc.FinishStat(&st)) { 887 if (st.st_size == static_cast<off_t>(ci.size)) { 888 // For links, we cannot update the atime/mtime. 889 if ((S_ISREG(ci.mode & st.st_mode) && st.st_mtime == ci.time) || 890 (S_ISLNK(ci.mode & st.st_mode) && st.st_mtime >= ci.time)) { 891 ci.skip = true; 892 } 893 } 894 } 895 } 896 } 897 898 sc.ComputeExpectedTotalBytes(file_list); 899 900 for (const copyinfo& ci : file_list) { 901 if (!ci.skip) { 902 if (list_only) { 903 sc.Println("would push: %s -> %s", ci.lpath.c_str(), ci.rpath.c_str()); 904 } else { 905 if (!sync_send(sc, ci.lpath.c_str(), ci.rpath.c_str(), ci.time, ci.mode)) { 906 return false; 907 } 908 } 909 } else { 910 skipped++; 911 } 912 } 913 914 sc.RecordFilesSkipped(skipped); 915 sc.ReportTransferRate(lpath, TransferDirection::push); 916 return true; 917 } 918 919 bool do_sync_push(const std::vector<const char*>& srcs, const char* dst) { 920 SyncConnection sc; 921 if (!sc.IsValid()) return false; 922 923 bool success = true; 924 bool dst_exists; 925 bool dst_isdir; 926 927 struct stat st; 928 if (sync_stat_fallback(sc, dst, &st)) { 929 dst_exists = true; 930 dst_isdir = S_ISDIR(st.st_mode); 931 } else { 932 if (errno == ENOENT || errno == ENOPROTOOPT) { 933 dst_exists = false; 934 dst_isdir = false; 935 } else { 936 sc.Error("stat failed when trying to push to %s: %s", dst, strerror(errno)); 937 return false; 938 } 939 } 940 941 if (!dst_isdir) { 942 if (srcs.size() > 1) { 943 sc.Error("target '%s' is not a directory", dst); 944 return false; 945 } else { 946 size_t dst_len = strlen(dst); 947 948 // A path that ends with a slash doesn't have to be a directory if 949 // it doesn't exist yet. 950 if (dst[dst_len - 1] == '/' && dst_exists) { 951 sc.Error("failed to access '%s': Not a directory", dst); 952 return false; 953 } 954 } 955 } 956 957 for (const char* src_path : srcs) { 958 const char* dst_path = dst; 959 struct stat st; 960 if (stat(src_path, &st) == -1) { 961 sc.Error("cannot stat '%s': %s", src_path, strerror(errno)); 962 success = false; 963 continue; 964 } 965 966 if (S_ISDIR(st.st_mode)) { 967 std::string dst_dir = dst; 968 969 // If the destination path existed originally, the source directory 970 // should be copied as a child of the destination. 971 if (dst_exists) { 972 if (!dst_isdir) { 973 sc.Error("target '%s' is not a directory", dst); 974 return false; 975 } 976 // dst is a POSIX path, so we don't want to use the sysdeps 977 // helpers here. 978 if (dst_dir.back() != '/') { 979 dst_dir.push_back('/'); 980 } 981 dst_dir.append(android::base::Basename(src_path)); 982 } 983 984 success &= copy_local_dir_remote(sc, src_path, dst_dir.c_str(), false, false); 985 continue; 986 } else if (!should_push_file(st.st_mode)) { 987 sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, st.st_mode); 988 continue; 989 } 990 991 std::string path_holder; 992 if (dst_isdir) { 993 // If we're copying a local file to a remote directory, 994 // we really want to copy to remote_dir + "/" + local_filename. 995 path_holder = dst_path; 996 if (path_holder.back() != '/') { 997 path_holder.push_back('/'); 998 } 999 path_holder += android::base::Basename(src_path); 1000 dst_path = path_holder.c_str(); 1001 } 1002 1003 sc.NewTransfer(); 1004 sc.SetExpectedTotalBytes(st.st_size); 1005 success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode); 1006 sc.ReportTransferRate(src_path, TransferDirection::push); 1007 } 1008 1009 sc.ReportOverallTransferRate(TransferDirection::push); 1010 return success; 1011 } 1012 1013 static bool remote_build_list(SyncConnection& sc, std::vector<copyinfo>* file_list, 1014 const std::string& rpath, const std::string& lpath) { 1015 std::vector<copyinfo> dirlist; 1016 std::vector<copyinfo> linklist; 1017 1018 // Add an entry for the current directory to ensure it gets created before pulling its contents. 1019 copyinfo ci(android::base::Dirname(lpath), android::base::Dirname(rpath), 1020 android::base::Basename(lpath), S_IFDIR); 1021 file_list->push_back(ci); 1022 1023 // Put the files/dirs in rpath on the lists. 1024 auto callback = [&](unsigned mode, unsigned size, unsigned time, const char* name) { 1025 if (IsDotOrDotDot(name)) { 1026 return; 1027 } 1028 1029 copyinfo ci(lpath, rpath, name, mode); 1030 if (S_ISDIR(mode)) { 1031 dirlist.push_back(ci); 1032 } else if (S_ISLNK(mode)) { 1033 linklist.push_back(ci); 1034 } else { 1035 if (!should_pull_file(ci.mode)) { 1036 sc.Warning("skipping special file '%s' (mode = 0o%o)", ci.rpath.c_str(), ci.mode); 1037 ci.skip = true; 1038 } 1039 ci.time = time; 1040 ci.size = size; 1041 file_list->push_back(ci); 1042 } 1043 }; 1044 1045 if (!sync_ls(sc, rpath.c_str(), callback)) { 1046 return false; 1047 } 1048 1049 // Check each symlink we found to see whether it's a file or directory. 1050 for (copyinfo& link_ci : linklist) { 1051 struct stat st; 1052 if (!sync_stat_fallback(sc, link_ci.rpath.c_str(), &st)) { 1053 sc.Warning("stat failed for path %s: %s", link_ci.rpath.c_str(), strerror(errno)); 1054 continue; 1055 } 1056 1057 if (S_ISDIR(st.st_mode)) { 1058 dirlist.emplace_back(std::move(link_ci)); 1059 } else { 1060 file_list->emplace_back(std::move(link_ci)); 1061 } 1062 } 1063 1064 // Recurse into each directory we found. 1065 while (!dirlist.empty()) { 1066 copyinfo current = dirlist.back(); 1067 dirlist.pop_back(); 1068 if (!remote_build_list(sc, file_list, current.rpath, current.lpath)) { 1069 return false; 1070 } 1071 } 1072 1073 return true; 1074 } 1075 1076 static int set_time_and_mode(const std::string& lpath, time_t time, 1077 unsigned int mode) { 1078 struct utimbuf times = { time, time }; 1079 int r1 = utime(lpath.c_str(), ×); 1080 1081 /* use umask for permissions */ 1082 mode_t mask = umask(0000); 1083 umask(mask); 1084 int r2 = chmod(lpath.c_str(), mode & ~mask); 1085 1086 return r1 ? r1 : r2; 1087 } 1088 1089 static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, 1090 std::string lpath, bool copy_attrs) { 1091 sc.NewTransfer(); 1092 1093 // Make sure that both directory paths end in a slash. 1094 // Both paths are known to be nonempty, so we don't need to check. 1095 ensure_trailing_separators(lpath, rpath); 1096 1097 // Recursively build the list of files to copy. 1098 sc.Printf("pull: building file list..."); 1099 std::vector<copyinfo> file_list; 1100 if (!remote_build_list(sc, &file_list, rpath.c_str(), lpath.c_str())) { 1101 return false; 1102 } 1103 1104 sc.ComputeExpectedTotalBytes(file_list); 1105 1106 int skipped = 0; 1107 for (const copyinfo &ci : file_list) { 1108 if (!ci.skip) { 1109 if (S_ISDIR(ci.mode)) { 1110 // Entry is for an empty directory, create it and continue. 1111 // TODO(b/25457350): We don't preserve permissions on directories. 1112 if (!mkdirs(ci.lpath)) { 1113 sc.Error("failed to create directory '%s': %s", 1114 ci.lpath.c_str(), strerror(errno)); 1115 return false; 1116 } 1117 continue; 1118 } 1119 1120 if (!sync_recv(sc, ci.rpath.c_str(), ci.lpath.c_str(), nullptr, ci.size)) { 1121 return false; 1122 } 1123 1124 if (copy_attrs && set_time_and_mode(ci.lpath, ci.time, ci.mode)) { 1125 return false; 1126 } 1127 } else { 1128 skipped++; 1129 } 1130 } 1131 1132 sc.RecordFilesSkipped(skipped); 1133 sc.ReportTransferRate(rpath, TransferDirection::pull); 1134 return true; 1135 } 1136 1137 bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, 1138 bool copy_attrs, const char* name) { 1139 SyncConnection sc; 1140 if (!sc.IsValid()) return false; 1141 1142 bool success = true; 1143 struct stat st; 1144 bool dst_exists = true; 1145 1146 if (stat(dst, &st) == -1) { 1147 dst_exists = false; 1148 1149 // If we're only pulling one path, the destination path might point to 1150 // a path that doesn't exist yet. 1151 if (srcs.size() == 1 && errno == ENOENT) { 1152 // However, its parent must exist. 1153 struct stat parent_st; 1154 if (stat(android::base::Dirname(dst).c_str(), &parent_st) == -1) { 1155 sc.Error("cannot create file/directory '%s': %s", dst, strerror(errno)); 1156 return false; 1157 } 1158 } else { 1159 sc.Error("failed to access '%s': %s", dst, strerror(errno)); 1160 return false; 1161 } 1162 } 1163 1164 bool dst_isdir = dst_exists && S_ISDIR(st.st_mode); 1165 if (!dst_isdir) { 1166 if (srcs.size() > 1) { 1167 sc.Error("target '%s' is not a directory", dst); 1168 return false; 1169 } else { 1170 size_t dst_len = strlen(dst); 1171 1172 // A path that ends with a slash doesn't have to be a directory if 1173 // it doesn't exist yet. 1174 if (adb_is_separator(dst[dst_len - 1]) && dst_exists) { 1175 sc.Error("failed to access '%s': Not a directory", dst); 1176 return false; 1177 } 1178 } 1179 } 1180 1181 for (const char* src_path : srcs) { 1182 const char* dst_path = dst; 1183 struct stat src_st; 1184 if (!sync_stat_fallback(sc, src_path, &src_st)) { 1185 if (errno == ENOPROTOOPT) { 1186 sc.Error("remote object '%s' does not exist", src_path); 1187 } else { 1188 sc.Error("failed to stat remote object '%s': %s", src_path, strerror(errno)); 1189 } 1190 1191 success = false; 1192 continue; 1193 } 1194 1195 bool src_isdir = S_ISDIR(src_st.st_mode); 1196 if (src_isdir) { 1197 std::string dst_dir = dst; 1198 1199 // If the destination path existed originally, the source directory 1200 // should be copied as a child of the destination. 1201 if (dst_exists) { 1202 if (!dst_isdir) { 1203 sc.Error("target '%s' is not a directory", dst); 1204 return false; 1205 } 1206 if (!adb_is_separator(dst_dir.back())) { 1207 dst_dir.push_back(OS_PATH_SEPARATOR); 1208 } 1209 dst_dir.append(android::base::Basename(src_path)); 1210 } 1211 1212 success &= copy_remote_dir_local(sc, src_path, dst_dir.c_str(), copy_attrs); 1213 continue; 1214 } else if (!should_pull_file(src_st.st_mode)) { 1215 sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, src_st.st_mode); 1216 continue; 1217 } 1218 1219 std::string path_holder; 1220 if (dst_isdir) { 1221 // If we're copying a remote file to a local directory, we 1222 // really want to copy to local_dir + OS_PATH_SEPARATOR + 1223 // basename(remote). 1224 path_holder = android::base::StringPrintf("%s%c%s", dst_path, OS_PATH_SEPARATOR, 1225 android::base::Basename(src_path).c_str()); 1226 dst_path = path_holder.c_str(); 1227 } 1228 1229 sc.NewTransfer(); 1230 sc.SetExpectedTotalBytes(src_st.st_size); 1231 if (!sync_recv(sc, src_path, dst_path, name, src_st.st_size)) { 1232 success = false; 1233 continue; 1234 } 1235 1236 if (copy_attrs && set_time_and_mode(dst_path, src_st.st_mtime, src_st.st_mode) != 0) { 1237 success = false; 1238 continue; 1239 } 1240 sc.ReportTransferRate(src_path, TransferDirection::pull); 1241 } 1242 1243 sc.ReportOverallTransferRate(TransferDirection::pull); 1244 return success; 1245 } 1246 1247 bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only) { 1248 SyncConnection sc; 1249 if (!sc.IsValid()) return false; 1250 1251 bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only); 1252 if (!list_only) { 1253 sc.ReportOverallTransferRate(TransferDirection::push); 1254 } 1255 return success; 1256 } 1257
