Home | History | Annotate | Download | only in dumpstate
      1 /*
      2  * Copyright (C) 2008 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #define LOG_TAG "dumpstate"
     18 
     19 #include "dumpstate.h"
     20 
     21 #include <dirent.h>
     22 #include <fcntl.h>
     23 #include <libgen.h>
     24 #include <math.h>
     25 #include <poll.h>
     26 #include <signal.h>
     27 #include <stdarg.h>
     28 #include <stdio.h>
     29 #include <stdlib.h>
     30 #include <string.h>
     31 #include <sys/capability.h>
     32 #include <sys/inotify.h>
     33 #include <sys/klog.h>
     34 #include <sys/prctl.h>
     35 #include <sys/stat.h>
     36 #include <sys/time.h>
     37 #include <sys/wait.h>
     38 #include <time.h>
     39 #include <unistd.h>
     40 
     41 #include <set>
     42 #include <string>
     43 #include <vector>
     44 
     45 #include <android-base/file.h>
     46 #include <android-base/properties.h>
     47 #include <android-base/stringprintf.h>
     48 #include <android-base/strings.h>
     49 #include <android/hidl/manager/1.0/IServiceManager.h>
     50 #include <cutils/properties.h>
     51 #include <cutils/sockets.h>
     52 #include <debuggerd/client.h>
     53 #include <log/log.h>
     54 #include <private/android_filesystem_config.h>
     55 
     56 #include "DumpstateInternal.h"
     57 
     58 // TODO: remove once moved to namespace
     59 using android::os::dumpstate::CommandOptions;
     60 using android::os::dumpstate::DumpFileToFd;
     61 using android::os::dumpstate::PropertiesHelper;
     62 
     63 // Keep in sync with
     64 // frameworks/base/services/core/java/com/android/server/am/ActivityManagerService.java
     65 static const int TRACE_DUMP_TIMEOUT_MS = 10000; // 10 seconds
     66 
     67 /* Most simple commands have 10 as timeout, so 5 is a good estimate */
     68 static const int32_t WEIGHT_FILE = 5;
     69 
     70 // TODO: temporary variables and functions used during C++ refactoring
     71 static Dumpstate& ds = Dumpstate::GetInstance();
     72 static int RunCommand(const std::string& title, const std::vector<std::string>& full_command,
     73                       const CommandOptions& options = CommandOptions::DEFAULT) {
     74     return ds.RunCommand(title, full_command, options);
     75 }
     76 
     77 /* list of native processes to include in the native dumps */
     78 // This matches the /proc/pid/exe link instead of /proc/pid/cmdline.
     79 static const char* native_processes_to_dump[] = {
     80         "/system/bin/audioserver",
     81         "/system/bin/cameraserver",
     82         "/system/bin/drmserver",
     83         "/system/bin/mediadrmserver",
     84         "/system/bin/mediaextractor", // media.extractor
     85         "/system/bin/mediaserver",
     86         "/system/bin/sdcard",
     87         "/system/bin/surfaceflinger",
     88         "/system/bin/vehicle_network_service",
     89         "/vendor/bin/hw/android.hardware.media.omx@1.0-service", // media.codec
     90         NULL,
     91 };
     92 
     93 /* list of hal interface to dump containing process during native dumps */
     94 static const char* hal_interfaces_to_dump[] {
     95         "android.hardware.audio (at) 2.0::IDevicesFactory",
     96         "android.hardware.bluetooth (at) 1.0::IBluetoothHci",
     97         "android.hardware.camera.provider (at) 2.4::ICameraProvider",
     98         "android.hardware.graphics.composer (at) 2.1::IComposer",
     99         "android.hardware.vr (at) 1.0::IVr",
    100         "android.hardware.media.omx (at) 1.0::IOmx",
    101         NULL,
    102 };
    103 
    104 // Reasonable value for max stats.
    105 static const int STATS_MAX_N_RUNS = 1000;
    106 static const long STATS_MAX_AVERAGE = 100000;
    107 
    108 CommandOptions Dumpstate::DEFAULT_DUMPSYS = CommandOptions::WithTimeout(30).Build();
    109 
    110 Dumpstate::Dumpstate(const std::string& version)
    111     : pid_(getpid()), version_(version), now_(time(nullptr)) {
    112 }
    113 
    114 Dumpstate& Dumpstate::GetInstance() {
    115     static Dumpstate singleton_(android::base::GetProperty("dumpstate.version", VERSION_CURRENT));
    116     return singleton_;
    117 }
    118 
    119 DurationReporter::DurationReporter(const std::string& title, bool log_only)
    120     : title_(title), log_only_(log_only) {
    121     if (!title_.empty()) {
    122         started_ = Nanotime();
    123     }
    124 }
    125 
    126 DurationReporter::~DurationReporter() {
    127     if (!title_.empty()) {
    128         uint64_t elapsed = Nanotime() - started_;
    129         if (log_only_) {
    130             MYLOGD("Duration of '%s': %.3fs\n", title_.c_str(), (float)elapsed / NANOS_PER_SEC);
    131         } else {
    132             // Use "Yoda grammar" to make it easier to grep|sort sections.
    133             printf("------ %.3fs was the duration of '%s' ------\n", (float)elapsed / NANOS_PER_SEC,
    134                    title_.c_str());
    135         }
    136     }
    137 }
    138 
    139 const int32_t Progress::kDefaultMax = 5000;
    140 
    141 Progress::Progress(const std::string& path) : Progress(Progress::kDefaultMax, 1.1, path) {
    142 }
    143 
    144 Progress::Progress(int32_t initial_max, int32_t progress, float growth_factor)
    145     : Progress(initial_max, growth_factor, "") {
    146     progress_ = progress;
    147 }
    148 
    149 Progress::Progress(int32_t initial_max, float growth_factor, const std::string& path)
    150     : initial_max_(initial_max),
    151       progress_(0),
    152       max_(initial_max),
    153       growth_factor_(growth_factor),
    154       n_runs_(0),
    155       average_max_(0),
    156       path_(path) {
    157     if (!path_.empty()) {
    158         Load();
    159     }
    160 }
    161 
    162 void Progress::Load() {
    163     MYLOGD("Loading stats from %s\n", path_.c_str());
    164     std::string content;
    165     if (!android::base::ReadFileToString(path_, &content)) {
    166         MYLOGI("Could not read stats from %s; using max of %d\n", path_.c_str(), max_);
    167         return;
    168     }
    169     if (content.empty()) {
    170         MYLOGE("No stats (empty file) on %s; using max of %d\n", path_.c_str(), max_);
    171         return;
    172     }
    173     std::vector<std::string> lines = android::base::Split(content, "\n");
    174 
    175     if (lines.size() < 1) {
    176         MYLOGE("Invalid stats on file %s: not enough lines (%d). Using max of %d\n", path_.c_str(),
    177                (int)lines.size(), max_);
    178         return;
    179     }
    180     char* ptr;
    181     n_runs_ = strtol(lines[0].c_str(), &ptr, 10);
    182     average_max_ = strtol(ptr, nullptr, 10);
    183     if (n_runs_ <= 0 || average_max_ <= 0 || n_runs_ > STATS_MAX_N_RUNS ||
    184         average_max_ > STATS_MAX_AVERAGE) {
    185         MYLOGE("Invalid stats line on file %s: %s\n", path_.c_str(), lines[0].c_str());
    186         initial_max_ = Progress::kDefaultMax;
    187     } else {
    188         initial_max_ = average_max_;
    189     }
    190     max_ = initial_max_;
    191 
    192     MYLOGI("Average max progress: %d in %d runs; estimated max: %d\n", average_max_, n_runs_, max_);
    193 }
    194 
    195 void Progress::Save() {
    196     int32_t total = n_runs_ * average_max_ + progress_;
    197     int32_t runs = n_runs_ + 1;
    198     int32_t average = floor(((float)total) / runs);
    199     MYLOGI("Saving stats (total=%d, runs=%d, average=%d) on %s\n", total, runs, average,
    200            path_.c_str());
    201     if (path_.empty()) {
    202         return;
    203     }
    204 
    205     std::string content = android::base::StringPrintf("%d %d\n", runs, average);
    206     if (!android::base::WriteStringToFile(content, path_)) {
    207         MYLOGE("Could not save stats on %s\n", path_.c_str());
    208     }
    209 }
    210 
    211 int32_t Progress::Get() const {
    212     return progress_;
    213 }
    214 
    215 bool Progress::Inc(int32_t delta) {
    216     bool changed = false;
    217     if (delta >= 0) {
    218         progress_ += delta;
    219         if (progress_ > max_) {
    220             int32_t old_max = max_;
    221             max_ = floor((float)progress_ * growth_factor_);
    222             MYLOGD("Adjusting max progress from %d to %d\n", old_max, max_);
    223             changed = true;
    224         }
    225     }
    226     return changed;
    227 }
    228 
    229 int32_t Progress::GetMax() const {
    230     return max_;
    231 }
    232 
    233 int32_t Progress::GetInitialMax() const {
    234     return initial_max_;
    235 }
    236 
    237 void Progress::Dump(int fd, const std::string& prefix) const {
    238     const char* pr = prefix.c_str();
    239     dprintf(fd, "%sprogress: %d\n", pr, progress_);
    240     dprintf(fd, "%smax: %d\n", pr, max_);
    241     dprintf(fd, "%sinitial_max: %d\n", pr, initial_max_);
    242     dprintf(fd, "%sgrowth_factor: %0.2f\n", pr, growth_factor_);
    243     dprintf(fd, "%spath: %s\n", pr, path_.c_str());
    244     dprintf(fd, "%sn_runs: %d\n", pr, n_runs_);
    245     dprintf(fd, "%saverage_max: %d\n", pr, average_max_);
    246 }
    247 
    248 bool Dumpstate::IsZipping() const {
    249     return zip_writer_ != nullptr;
    250 }
    251 
    252 std::string Dumpstate::GetPath(const std::string& suffix) const {
    253     return android::base::StringPrintf("%s/%s-%s%s", bugreport_dir_.c_str(), base_name_.c_str(),
    254                                        name_.c_str(), suffix.c_str());
    255 }
    256 
    257 void Dumpstate::SetProgress(std::unique_ptr<Progress> progress) {
    258     progress_ = std::move(progress);
    259 }
    260 
    261 void for_each_userid(void (*func)(int), const char *header) {
    262     std::string title = header == nullptr ? "for_each_userid" : android::base::StringPrintf(
    263                                                                     "for_each_userid(%s)", header);
    264     DurationReporter duration_reporter(title);
    265     if (PropertiesHelper::IsDryRun()) return;
    266 
    267     DIR *d;
    268     struct dirent *de;
    269 
    270     if (header) printf("\n------ %s ------\n", header);
    271     func(0);
    272 
    273     if (!(d = opendir("/data/system/users"))) {
    274         printf("Failed to open /data/system/users (%s)\n", strerror(errno));
    275         return;
    276     }
    277 
    278     while ((de = readdir(d))) {
    279         int userid;
    280         if (de->d_type != DT_DIR || !(userid = atoi(de->d_name))) {
    281             continue;
    282         }
    283         func(userid);
    284     }
    285 
    286     closedir(d);
    287 }
    288 
    289 static void __for_each_pid(void (*helper)(int, const char *, void *), const char *header, void *arg) {
    290     DIR *d;
    291     struct dirent *de;
    292 
    293     if (!(d = opendir("/proc"))) {
    294         printf("Failed to open /proc (%s)\n", strerror(errno));
    295         return;
    296     }
    297 
    298     if (header) printf("\n------ %s ------\n", header);
    299     while ((de = readdir(d))) {
    300         int pid;
    301         int fd;
    302         char cmdpath[255];
    303         char cmdline[255];
    304 
    305         if (!(pid = atoi(de->d_name))) {
    306             continue;
    307         }
    308 
    309         memset(cmdline, 0, sizeof(cmdline));
    310 
    311         snprintf(cmdpath, sizeof(cmdpath), "/proc/%d/cmdline", pid);
    312         if ((fd = TEMP_FAILURE_RETRY(open(cmdpath, O_RDONLY | O_CLOEXEC))) >= 0) {
    313             TEMP_FAILURE_RETRY(read(fd, cmdline, sizeof(cmdline) - 2));
    314             close(fd);
    315             if (cmdline[0]) {
    316                 helper(pid, cmdline, arg);
    317                 continue;
    318             }
    319         }
    320 
    321         // if no cmdline, a kernel thread has comm
    322         snprintf(cmdpath, sizeof(cmdpath), "/proc/%d/comm", pid);
    323         if ((fd = TEMP_FAILURE_RETRY(open(cmdpath, O_RDONLY | O_CLOEXEC))) >= 0) {
    324             TEMP_FAILURE_RETRY(read(fd, cmdline + 1, sizeof(cmdline) - 4));
    325             close(fd);
    326             if (cmdline[1]) {
    327                 cmdline[0] = '[';
    328                 size_t len = strcspn(cmdline, "\f\b\r\n");
    329                 cmdline[len] = ']';
    330                 cmdline[len+1] = '\0';
    331             }
    332         }
    333         if (!cmdline[0]) {
    334             strcpy(cmdline, "N/A");
    335         }
    336         helper(pid, cmdline, arg);
    337     }
    338 
    339     closedir(d);
    340 }
    341 
    342 static void for_each_pid_helper(int pid, const char *cmdline, void *arg) {
    343     for_each_pid_func *func = (for_each_pid_func*) arg;
    344     func(pid, cmdline);
    345 }
    346 
    347 void for_each_pid(for_each_pid_func func, const char *header) {
    348     std::string title = header == nullptr ? "for_each_pid"
    349                                           : android::base::StringPrintf("for_each_pid(%s)", header);
    350     DurationReporter duration_reporter(title);
    351     if (PropertiesHelper::IsDryRun()) return;
    352 
    353     __for_each_pid(for_each_pid_helper, header, (void *) func);
    354 }
    355 
    356 static void for_each_tid_helper(int pid, const char *cmdline, void *arg) {
    357     DIR *d;
    358     struct dirent *de;
    359     char taskpath[255];
    360     for_each_tid_func *func = (for_each_tid_func *) arg;
    361 
    362     snprintf(taskpath, sizeof(taskpath), "/proc/%d/task", pid);
    363 
    364     if (!(d = opendir(taskpath))) {
    365         printf("Failed to open %s (%s)\n", taskpath, strerror(errno));
    366         return;
    367     }
    368 
    369     func(pid, pid, cmdline);
    370 
    371     while ((de = readdir(d))) {
    372         int tid;
    373         int fd;
    374         char commpath[255];
    375         char comm[255];
    376 
    377         if (!(tid = atoi(de->d_name))) {
    378             continue;
    379         }
    380 
    381         if (tid == pid)
    382             continue;
    383 
    384         snprintf(commpath, sizeof(commpath), "/proc/%d/comm", tid);
    385         memset(comm, 0, sizeof(comm));
    386         if ((fd = TEMP_FAILURE_RETRY(open(commpath, O_RDONLY | O_CLOEXEC))) < 0) {
    387             strcpy(comm, "N/A");
    388         } else {
    389             char *c;
    390             TEMP_FAILURE_RETRY(read(fd, comm, sizeof(comm) - 2));
    391             close(fd);
    392 
    393             c = strrchr(comm, '\n');
    394             if (c) {
    395                 *c = '\0';
    396             }
    397         }
    398         func(pid, tid, comm);
    399     }
    400 
    401     closedir(d);
    402 }
    403 
    404 void for_each_tid(for_each_tid_func func, const char *header) {
    405     std::string title = header == nullptr ? "for_each_tid"
    406                                           : android::base::StringPrintf("for_each_tid(%s)", header);
    407     DurationReporter duration_reporter(title);
    408 
    409     if (PropertiesHelper::IsDryRun()) return;
    410 
    411     __for_each_pid(for_each_tid_helper, header, (void *) func);
    412 }
    413 
    414 void show_wchan(int pid, int tid, const char *name) {
    415     if (PropertiesHelper::IsDryRun()) return;
    416 
    417     char path[255];
    418     char buffer[255];
    419     int fd, ret, save_errno;
    420     char name_buffer[255];
    421 
    422     memset(buffer, 0, sizeof(buffer));
    423 
    424     snprintf(path, sizeof(path), "/proc/%d/wchan", tid);
    425     if ((fd = TEMP_FAILURE_RETRY(open(path, O_RDONLY | O_CLOEXEC))) < 0) {
    426         printf("Failed to open '%s' (%s)\n", path, strerror(errno));
    427         return;
    428     }
    429 
    430     ret = TEMP_FAILURE_RETRY(read(fd, buffer, sizeof(buffer)));
    431     save_errno = errno;
    432     close(fd);
    433 
    434     if (ret < 0) {
    435         printf("Failed to read '%s' (%s)\n", path, strerror(save_errno));
    436         return;
    437     }
    438 
    439     snprintf(name_buffer, sizeof(name_buffer), "%*s%s",
    440              pid == tid ? 0 : 3, "", name);
    441 
    442     printf("%-7d %-32s %s\n", tid, name_buffer, buffer);
    443 
    444     return;
    445 }
    446 
    447 // print time in centiseconds
    448 static void snprcent(char *buffer, size_t len, size_t spc,
    449                      unsigned long long time) {
    450     static long hz; // cache discovered hz
    451 
    452     if (hz <= 0) {
    453         hz = sysconf(_SC_CLK_TCK);
    454         if (hz <= 0) {
    455             hz = 1000;
    456         }
    457     }
    458 
    459     // convert to centiseconds
    460     time = (time * 100 + (hz / 2)) / hz;
    461 
    462     char str[16];
    463 
    464     snprintf(str, sizeof(str), " %llu.%02u",
    465              time / 100, (unsigned)(time % 100));
    466     size_t offset = strlen(buffer);
    467     snprintf(buffer + offset, (len > offset) ? len - offset : 0,
    468              "%*s", (spc > offset) ? (int)(spc - offset) : 0, str);
    469 }
    470 
    471 // print permille as a percent
    472 static void snprdec(char *buffer, size_t len, size_t spc, unsigned permille) {
    473     char str[16];
    474 
    475     snprintf(str, sizeof(str), " %u.%u%%", permille / 10, permille % 10);
    476     size_t offset = strlen(buffer);
    477     snprintf(buffer + offset, (len > offset) ? len - offset : 0,
    478              "%*s", (spc > offset) ? (int)(spc - offset) : 0, str);
    479 }
    480 
    481 void show_showtime(int pid, const char *name) {
    482     if (PropertiesHelper::IsDryRun()) return;
    483 
    484     char path[255];
    485     char buffer[1023];
    486     int fd, ret, save_errno;
    487 
    488     memset(buffer, 0, sizeof(buffer));
    489 
    490     snprintf(path, sizeof(path), "/proc/%d/stat", pid);
    491     if ((fd = TEMP_FAILURE_RETRY(open(path, O_RDONLY | O_CLOEXEC))) < 0) {
    492         printf("Failed to open '%s' (%s)\n", path, strerror(errno));
    493         return;
    494     }
    495 
    496     ret = TEMP_FAILURE_RETRY(read(fd, buffer, sizeof(buffer)));
    497     save_errno = errno;
    498     close(fd);
    499 
    500     if (ret < 0) {
    501         printf("Failed to read '%s' (%s)\n", path, strerror(save_errno));
    502         return;
    503     }
    504 
    505     // field 14 is utime
    506     // field 15 is stime
    507     // field 42 is iotime
    508     unsigned long long utime = 0, stime = 0, iotime = 0;
    509     if (sscanf(buffer,
    510                "%*u %*s %*s %*d %*d %*d %*d %*d %*d %*d %*d "
    511                "%*d %*d %llu %llu %*d %*d %*d %*d %*d %*d "
    512                "%*d %*d %*d %*d %*d %*d %*d %*d %*d %*d "
    513                "%*d %*d %*d %*d %*d %*d %*d %*d %*d %llu ",
    514                &utime, &stime, &iotime) != 3) {
    515         return;
    516     }
    517 
    518     unsigned long long total = utime + stime;
    519     if (!total) {
    520         return;
    521     }
    522 
    523     unsigned permille = (iotime * 1000 + (total / 2)) / total;
    524     if (permille > 1000) {
    525         permille = 1000;
    526     }
    527 
    528     // try to beautify and stabilize columns at <80 characters
    529     snprintf(buffer, sizeof(buffer), "%-6d%s", pid, name);
    530     if ((name[0] != '[') || utime) {
    531         snprcent(buffer, sizeof(buffer), 57, utime);
    532     }
    533     snprcent(buffer, sizeof(buffer), 65, stime);
    534     if ((name[0] != '[') || iotime) {
    535         snprcent(buffer, sizeof(buffer), 73, iotime);
    536     }
    537     if (iotime) {
    538         snprdec(buffer, sizeof(buffer), 79, permille);
    539     }
    540     puts(buffer);  // adds a trailing newline
    541 
    542     return;
    543 }
    544 
    545 void do_dmesg() {
    546     const char *title = "KERNEL LOG (dmesg)";
    547     DurationReporter duration_reporter(title);
    548     printf("------ %s ------\n", title);
    549 
    550     if (PropertiesHelper::IsDryRun()) return;
    551 
    552     /* Get size of kernel buffer */
    553     int size = klogctl(KLOG_SIZE_BUFFER, NULL, 0);
    554     if (size <= 0) {
    555         printf("Unexpected klogctl return value: %d\n\n", size);
    556         return;
    557     }
    558     char *buf = (char *) malloc(size + 1);
    559     if (buf == NULL) {
    560         printf("memory allocation failed\n\n");
    561         return;
    562     }
    563     int retval = klogctl(KLOG_READ_ALL, buf, size);
    564     if (retval < 0) {
    565         printf("klogctl failure\n\n");
    566         free(buf);
    567         return;
    568     }
    569     buf[retval] = '\0';
    570     printf("%s\n\n", buf);
    571     free(buf);
    572     return;
    573 }
    574 
    575 void do_showmap(int pid, const char *name) {
    576     char title[255];
    577     char arg[255];
    578 
    579     snprintf(title, sizeof(title), "SHOW MAP %d (%s)", pid, name);
    580     snprintf(arg, sizeof(arg), "%d", pid);
    581     RunCommand(title, {"showmap", "-q", arg}, CommandOptions::AS_ROOT);
    582 }
    583 
    584 int Dumpstate::DumpFile(const std::string& title, const std::string& path) {
    585     DurationReporter duration_reporter(title);
    586 
    587     int status = DumpFileToFd(STDOUT_FILENO, title, path);
    588 
    589     UpdateProgress(WEIGHT_FILE);
    590 
    591     return status;
    592 }
    593 
    594 int read_file_as_long(const char *path, long int *output) {
    595     int fd = TEMP_FAILURE_RETRY(open(path, O_RDONLY | O_NONBLOCK | O_CLOEXEC));
    596     if (fd < 0) {
    597         int err = errno;
    598         MYLOGE("Error opening file descriptor for %s: %s\n", path, strerror(err));
    599         return -1;
    600     }
    601     char buffer[50];
    602     ssize_t bytes_read = TEMP_FAILURE_RETRY(read(fd, buffer, sizeof(buffer)));
    603     if (bytes_read == -1) {
    604         MYLOGE("Error reading file %s: %s\n", path, strerror(errno));
    605         return -2;
    606     }
    607     if (bytes_read == 0) {
    608         MYLOGE("File %s is empty\n", path);
    609         return -3;
    610     }
    611     *output = atoi(buffer);
    612     return 0;
    613 }
    614 
    615 /* calls skip to gate calling dump_from_fd recursively
    616  * in the specified directory. dump_from_fd defaults to
    617  * dump_file_from_fd above when set to NULL. skip defaults
    618  * to false when set to NULL. dump_from_fd will always be
    619  * called with title NULL.
    620  */
    621 int dump_files(const std::string& title, const char* dir, bool (*skip)(const char* path),
    622                int (*dump_from_fd)(const char* title, const char* path, int fd)) {
    623     DurationReporter duration_reporter(title);
    624     DIR *dirp;
    625     struct dirent *d;
    626     char *newpath = NULL;
    627     const char *slash = "/";
    628     int fd, retval = 0;
    629 
    630     if (!title.empty()) {
    631         printf("------ %s (%s) ------\n", title.c_str(), dir);
    632     }
    633     if (PropertiesHelper::IsDryRun()) return 0;
    634 
    635     if (dir[strlen(dir) - 1] == '/') {
    636         ++slash;
    637     }
    638     dirp = opendir(dir);
    639     if (dirp == NULL) {
    640         retval = -errno;
    641         MYLOGE("%s: %s\n", dir, strerror(errno));
    642         return retval;
    643     }
    644 
    645     if (!dump_from_fd) {
    646         dump_from_fd = dump_file_from_fd;
    647     }
    648     for (; ((d = readdir(dirp))); free(newpath), newpath = NULL) {
    649         if ((d->d_name[0] == '.')
    650          && (((d->d_name[1] == '.') && (d->d_name[2] == '\0'))
    651           || (d->d_name[1] == '\0'))) {
    652             continue;
    653         }
    654         asprintf(&newpath, "%s%s%s%s", dir, slash, d->d_name,
    655                  (d->d_type == DT_DIR) ? "/" : "");
    656         if (!newpath) {
    657             retval = -errno;
    658             continue;
    659         }
    660         if (skip && (*skip)(newpath)) {
    661             continue;
    662         }
    663         if (d->d_type == DT_DIR) {
    664             int ret = dump_files("", newpath, skip, dump_from_fd);
    665             if (ret < 0) {
    666                 retval = ret;
    667             }
    668             continue;
    669         }
    670         fd = TEMP_FAILURE_RETRY(open(newpath, O_RDONLY | O_NONBLOCK | O_CLOEXEC));
    671         if (fd < 0) {
    672             retval = fd;
    673             printf("*** %s: %s\n", newpath, strerror(errno));
    674             continue;
    675         }
    676         (*dump_from_fd)(NULL, newpath, fd);
    677     }
    678     closedir(dirp);
    679     if (!title.empty()) {
    680         printf("\n");
    681     }
    682     return retval;
    683 }
    684 
    685 /* fd must have been opened with the flag O_NONBLOCK. With this flag set,
    686  * it's possible to avoid issues where opening the file itself can get
    687  * stuck.
    688  */
    689 int dump_file_from_fd(const char *title, const char *path, int fd) {
    690     if (PropertiesHelper::IsDryRun()) return 0;
    691 
    692     int flags = fcntl(fd, F_GETFL);
    693     if (flags == -1) {
    694         printf("*** %s: failed to get flags on fd %d: %s\n", path, fd, strerror(errno));
    695         close(fd);
    696         return -1;
    697     } else if (!(flags & O_NONBLOCK)) {
    698         printf("*** %s: fd must have O_NONBLOCK set.\n", path);
    699         close(fd);
    700         return -1;
    701     }
    702     return DumpFileFromFdToFd(title, path, fd, STDOUT_FILENO, PropertiesHelper::IsDryRun());
    703 }
    704 
    705 int Dumpstate::RunCommand(const std::string& title, const std::vector<std::string>& full_command,
    706                           const CommandOptions& options) {
    707     DurationReporter duration_reporter(title);
    708 
    709     int status = RunCommandToFd(STDOUT_FILENO, title, full_command, options);
    710 
    711     /* TODO: for now we're simplifying the progress calculation by using the
    712      * timeout as the weight. It's a good approximation for most cases, except when calling dumpsys,
    713      * where its weight should be much higher proportionally to its timeout.
    714      * Ideally, it should use a options.EstimatedDuration() instead...*/
    715     UpdateProgress(options.Timeout());
    716 
    717     return status;
    718 }
    719 
    720 void Dumpstate::RunDumpsys(const std::string& title, const std::vector<std::string>& dumpsys_args,
    721                            const CommandOptions& options, long dumpsysTimeout) {
    722     long timeout = dumpsysTimeout > 0 ? dumpsysTimeout : options.Timeout();
    723     std::vector<std::string> dumpsys = {"/system/bin/dumpsys", "-t", std::to_string(timeout)};
    724     dumpsys.insert(dumpsys.end(), dumpsys_args.begin(), dumpsys_args.end());
    725     RunCommand(title, dumpsys, options);
    726 }
    727 
    728 int open_socket(const char *service) {
    729     int s = android_get_control_socket(service);
    730     if (s < 0) {
    731         MYLOGE("android_get_control_socket(%s): %s\n", service, strerror(errno));
    732         exit(1);
    733     }
    734     fcntl(s, F_SETFD, FD_CLOEXEC);
    735     if (listen(s, 4) < 0) {
    736         MYLOGE("listen(control socket): %s\n", strerror(errno));
    737         exit(1);
    738     }
    739 
    740     struct sockaddr addr;
    741     socklen_t alen = sizeof(addr);
    742     int fd = accept(s, &addr, &alen);
    743     if (fd < 0) {
    744         MYLOGE("accept(control socket): %s\n", strerror(errno));
    745         exit(1);
    746     }
    747 
    748     return fd;
    749 }
    750 
    751 /* redirect output to a service control socket */
    752 void redirect_to_socket(FILE *redirect, const char *service) {
    753     int fd = open_socket(service);
    754     fflush(redirect);
    755     dup2(fd, fileno(redirect));
    756     close(fd);
    757 }
    758 
    759 // TODO: should call is_valid_output_file and/or be merged into it.
    760 void create_parent_dirs(const char *path) {
    761     char *chp = const_cast<char *> (path);
    762 
    763     /* skip initial slash */
    764     if (chp[0] == '/')
    765         chp++;
    766 
    767     /* create leading directories, if necessary */
    768     struct stat dir_stat;
    769     while (chp && chp[0]) {
    770         chp = strchr(chp, '/');
    771         if (chp) {
    772             *chp = 0;
    773             if (stat(path, &dir_stat) == -1 || !S_ISDIR(dir_stat.st_mode)) {
    774                 MYLOGI("Creating directory %s\n", path);
    775                 if (mkdir(path, 0770)) { /* drwxrwx--- */
    776                     MYLOGE("Unable to create directory %s: %s\n", path, strerror(errno));
    777                 } else if (chown(path, AID_SHELL, AID_SHELL)) {
    778                     MYLOGE("Unable to change ownership of dir %s: %s\n", path, strerror(errno));
    779                 }
    780             }
    781             *chp++ = '/';
    782         }
    783     }
    784 }
    785 
    786 void _redirect_to_file(FILE *redirect, char *path, int truncate_flag) {
    787     create_parent_dirs(path);
    788 
    789     int fd = TEMP_FAILURE_RETRY(open(path,
    790                                      O_WRONLY | O_CREAT | truncate_flag | O_CLOEXEC | O_NOFOLLOW,
    791                                      S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH));
    792     if (fd < 0) {
    793         MYLOGE("%s: %s\n", path, strerror(errno));
    794         exit(1);
    795     }
    796 
    797     TEMP_FAILURE_RETRY(dup2(fd, fileno(redirect)));
    798     close(fd);
    799 }
    800 
    801 void redirect_to_file(FILE *redirect, char *path) {
    802     _redirect_to_file(redirect, path, O_TRUNC);
    803 }
    804 
    805 void redirect_to_existing_file(FILE *redirect, char *path) {
    806     _redirect_to_file(redirect, path, O_APPEND);
    807 }
    808 
    809 static bool should_dump_hal_interface(const char* interface) {
    810     for (const char** i = hal_interfaces_to_dump; *i; i++) {
    811         if (!strcmp(*i, interface)) {
    812             return true;
    813         }
    814     }
    815     return false;
    816 }
    817 
    818 static bool should_dump_native_traces(const char* path) {
    819     for (const char** p = native_processes_to_dump; *p; p++) {
    820         if (!strcmp(*p, path)) {
    821             return true;
    822         }
    823     }
    824     return false;
    825 }
    826 
    827 std::set<int> get_interesting_hal_pids() {
    828     using android::hidl::manager::V1_0::IServiceManager;
    829     using android::sp;
    830     using android::hardware::Return;
    831 
    832     sp<IServiceManager> manager = IServiceManager::getService();
    833     std::set<int> pids;
    834 
    835     Return<void> ret = manager->debugDump([&](auto& hals) {
    836         for (const auto &info : hals) {
    837             if (info.pid == static_cast<int>(IServiceManager::PidConstant::NO_PID)) {
    838                 continue;
    839             }
    840 
    841             if (!should_dump_hal_interface(info.interfaceName.c_str())) {
    842                 continue;
    843             }
    844 
    845             pids.insert(info.pid);
    846         }
    847     });
    848 
    849     if (!ret.isOk()) {
    850         MYLOGE("Could not get list of HAL PIDs: %s\n", ret.description().c_str());
    851     }
    852 
    853     return pids; // whether it was okay or not
    854 }
    855 
    856 /* dump Dalvik and native stack traces, return the trace file location (NULL if none) */
    857 const char *dump_traces() {
    858     DurationReporter duration_reporter("DUMP TRACES");
    859 
    860     const char* result = nullptr;
    861 
    862     std::string traces_path = android::base::GetProperty("dalvik.vm.stack-trace-file", "");
    863     if (traces_path.empty()) return nullptr;
    864 
    865     /* move the old traces.txt (if any) out of the way temporarily */
    866     std::string anrtraces_path = traces_path + ".anr";
    867     if (rename(traces_path.c_str(), anrtraces_path.c_str()) && errno != ENOENT) {
    868         MYLOGE("rename(%s, %s): %s\n", traces_path.c_str(), anrtraces_path.c_str(), strerror(errno));
    869         return nullptr;  // Can't rename old traces.txt -- no permission? -- leave it alone instead
    870     }
    871 
    872     /* create a new, empty traces.txt file to receive stack dumps */
    873     int fd = TEMP_FAILURE_RETRY(
    874         open(traces_path.c_str(), O_CREAT | O_WRONLY | O_APPEND | O_TRUNC | O_NOFOLLOW | O_CLOEXEC,
    875              0666)); /* -rw-rw-rw- */
    876     if (fd < 0) {
    877         MYLOGE("%s: %s\n", traces_path.c_str(), strerror(errno));
    878         return nullptr;
    879     }
    880     int chmod_ret = fchmod(fd, 0666);
    881     if (chmod_ret < 0) {
    882         MYLOGE("fchmod on %s failed: %s\n", traces_path.c_str(), strerror(errno));
    883         close(fd);
    884         return nullptr;
    885     }
    886 
    887     /* Variables below must be initialized before 'goto' statements */
    888     int dalvik_found = 0;
    889     int ifd, wfd = -1;
    890     std::set<int> hal_pids = get_interesting_hal_pids();
    891 
    892     /* walk /proc and kill -QUIT all Dalvik processes */
    893     DIR *proc = opendir("/proc");
    894     if (proc == NULL) {
    895         MYLOGE("/proc: %s\n", strerror(errno));
    896         goto error_close_fd;
    897     }
    898 
    899     /* use inotify to find when processes are done dumping */
    900     ifd = inotify_init();
    901     if (ifd < 0) {
    902         MYLOGE("inotify_init: %s\n", strerror(errno));
    903         goto error_close_fd;
    904     }
    905 
    906     wfd = inotify_add_watch(ifd, traces_path.c_str(), IN_CLOSE_WRITE);
    907     if (wfd < 0) {
    908         MYLOGE("inotify_add_watch(%s): %s\n", traces_path.c_str(), strerror(errno));
    909         goto error_close_ifd;
    910     }
    911 
    912     struct dirent *d;
    913     while ((d = readdir(proc))) {
    914         int pid = atoi(d->d_name);
    915         if (pid <= 0) continue;
    916 
    917         char path[PATH_MAX];
    918         char data[PATH_MAX];
    919         snprintf(path, sizeof(path), "/proc/%d/exe", pid);
    920         ssize_t len = readlink(path, data, sizeof(data) - 1);
    921         if (len <= 0) {
    922             continue;
    923         }
    924         data[len] = '\0';
    925 
    926         if (!strncmp(data, "/system/bin/app_process", strlen("/system/bin/app_process"))) {
    927             /* skip zygote -- it won't dump its stack anyway */
    928             snprintf(path, sizeof(path), "/proc/%d/cmdline", pid);
    929             int cfd = TEMP_FAILURE_RETRY(open(path, O_RDONLY | O_CLOEXEC));
    930             len = read(cfd, data, sizeof(data) - 1);
    931             close(cfd);
    932             if (len <= 0) {
    933                 continue;
    934             }
    935             data[len] = '\0';
    936             if (!strncmp(data, "zygote", strlen("zygote"))) {
    937                 continue;
    938             }
    939 
    940             ++dalvik_found;
    941             uint64_t start = Nanotime();
    942             if (kill(pid, SIGQUIT)) {
    943                 MYLOGE("kill(%d, SIGQUIT): %s\n", pid, strerror(errno));
    944                 continue;
    945             }
    946 
    947             /* wait for the writable-close notification from inotify */
    948             struct pollfd pfd = { ifd, POLLIN, 0 };
    949             int ret = poll(&pfd, 1, TRACE_DUMP_TIMEOUT_MS);
    950             if (ret < 0) {
    951                 MYLOGE("poll: %s\n", strerror(errno));
    952             } else if (ret == 0) {
    953                 MYLOGE("warning: timed out dumping pid %d\n", pid);
    954             } else {
    955                 struct inotify_event ie;
    956                 read(ifd, &ie, sizeof(ie));
    957             }
    958 
    959             if (lseek(fd, 0, SEEK_END) < 0) {
    960                 MYLOGE("lseek: %s\n", strerror(errno));
    961             } else {
    962                 dprintf(fd, "[dump dalvik stack %d: %.3fs elapsed]\n", pid,
    963                         (float)(Nanotime() - start) / NANOS_PER_SEC);
    964             }
    965         } else if (should_dump_native_traces(data) ||
    966                    hal_pids.find(pid) != hal_pids.end()) {
    967             /* dump native process if appropriate */
    968             if (lseek(fd, 0, SEEK_END) < 0) {
    969                 MYLOGE("lseek: %s\n", strerror(errno));
    970             } else {
    971                 static uint16_t timeout_failures = 0;
    972                 uint64_t start = Nanotime();
    973 
    974                 /* If 3 backtrace dumps fail in a row, consider debuggerd dead. */
    975                 if (timeout_failures == 3) {
    976                     dprintf(fd, "too many stack dump failures, skipping...\n");
    977                 } else if (dump_backtrace_to_file_timeout(pid, fd, 20) == -1) {
    978                     dprintf(fd, "dumping failed, likely due to a timeout\n");
    979                     timeout_failures++;
    980                 } else {
    981                     timeout_failures = 0;
    982                 }
    983                 dprintf(fd, "[dump native stack %d: %.3fs elapsed]\n", pid,
    984                         (float)(Nanotime() - start) / NANOS_PER_SEC);
    985             }
    986         }
    987     }
    988 
    989     if (dalvik_found == 0) {
    990         MYLOGE("Warning: no Dalvik processes found to dump stacks\n");
    991     }
    992 
    993     static std::string dumptraces_path = android::base::StringPrintf(
    994         "%s/bugreport-%s", dirname(traces_path.c_str()), basename(traces_path.c_str()));
    995     if (rename(traces_path.c_str(), dumptraces_path.c_str())) {
    996         MYLOGE("rename(%s, %s): %s\n", traces_path.c_str(), dumptraces_path.c_str(),
    997                strerror(errno));
    998         goto error_close_ifd;
    999     }
   1000     result = dumptraces_path.c_str();
   1001 
   1002     /* replace the saved [ANR] traces.txt file */
   1003     rename(anrtraces_path.c_str(), traces_path.c_str());
   1004 
   1005 error_close_ifd:
   1006     close(ifd);
   1007 error_close_fd:
   1008     close(fd);
   1009     return result;
   1010 }
   1011 
   1012 void dump_route_tables() {
   1013     DurationReporter duration_reporter("DUMP ROUTE TABLES");
   1014     if (PropertiesHelper::IsDryRun()) return;
   1015     const char* const RT_TABLES_PATH = "/data/misc/net/rt_tables";
   1016     ds.DumpFile("RT_TABLES", RT_TABLES_PATH);
   1017     FILE* fp = fopen(RT_TABLES_PATH, "re");
   1018     if (!fp) {
   1019         printf("*** %s: %s\n", RT_TABLES_PATH, strerror(errno));
   1020         return;
   1021     }
   1022     char table[16];
   1023     // Each line has an integer (the table number), a space, and a string (the table name). We only
   1024     // need the table number. It's a 32-bit unsigned number, so max 10 chars. Skip the table name.
   1025     // Add a fixed max limit so this doesn't go awry.
   1026     for (int i = 0; i < 64 && fscanf(fp, " %10s %*s", table) == 1; ++i) {
   1027         RunCommand("ROUTE TABLE IPv4", {"ip", "-4", "route", "show", "table", table});
   1028         RunCommand("ROUTE TABLE IPv6", {"ip", "-6", "route", "show", "table", table});
   1029     }
   1030     fclose(fp);
   1031 }
   1032 
   1033 // TODO: make this function thread safe if sections are generated in parallel.
   1034 void Dumpstate::UpdateProgress(int32_t delta) {
   1035     if (progress_ == nullptr) {
   1036         MYLOGE("UpdateProgress: progress_ not set\n");
   1037         return;
   1038     }
   1039 
   1040     // Always update progess so stats can be tuned...
   1041     bool max_changed = progress_->Inc(delta);
   1042 
   1043     // ...but only notifiy listeners when necessary.
   1044     if (!update_progress_) return;
   1045 
   1046     int progress = progress_->Get();
   1047     int max = progress_->GetMax();
   1048 
   1049     // adjusts max on the fly
   1050     if (max_changed && listener_ != nullptr) {
   1051         listener_->onMaxProgressUpdated(max);
   1052     }
   1053 
   1054     int32_t last_update_delta = progress - last_updated_progress_;
   1055     if (last_updated_progress_ > 0 && last_update_delta < update_progress_threshold_) {
   1056         return;
   1057     }
   1058     last_updated_progress_ = progress;
   1059 
   1060     if (control_socket_fd_ >= 0) {
   1061         dprintf(control_socket_fd_, "PROGRESS:%d/%d\n", progress, max);
   1062         fsync(control_socket_fd_);
   1063     }
   1064 
   1065     if (listener_ != nullptr) {
   1066         if (progress % 100 == 0) {
   1067             // We don't want to spam logcat, so only log multiples of 100.
   1068             MYLOGD("Setting progress (%s): %d/%d\n", listener_name_.c_str(), progress, max);
   1069         } else {
   1070             // stderr is ignored on normal invocations, but useful when calling
   1071             // /system/bin/dumpstate directly for debuggging.
   1072             fprintf(stderr, "Setting progress (%s): %d/%d\n", listener_name_.c_str(), progress, max);
   1073         }
   1074         listener_->onProgressUpdated(progress);
   1075     }
   1076 }
   1077 
   1078 void Dumpstate::TakeScreenshot(const std::string& path) {
   1079     const std::string& real_path = path.empty() ? screenshot_path_ : path;
   1080     int status =
   1081         RunCommand("", {"/system/bin/screencap", "-p", real_path},
   1082                    CommandOptions::WithTimeout(10).Always().DropRoot().RedirectStderr().Build());
   1083     if (status == 0) {
   1084         MYLOGD("Screenshot saved on %s\n", real_path.c_str());
   1085     } else {
   1086         MYLOGE("Failed to take screenshot on %s\n", real_path.c_str());
   1087     }
   1088 }
   1089 
   1090 bool is_dir(const char* pathname) {
   1091     struct stat info;
   1092     if (stat(pathname, &info) == -1) {
   1093         return false;
   1094     }
   1095     return S_ISDIR(info.st_mode);
   1096 }
   1097 
   1098 time_t get_mtime(int fd, time_t default_mtime) {
   1099     struct stat info;
   1100     if (fstat(fd, &info) == -1) {
   1101         return default_mtime;
   1102     }
   1103     return info.st_mtime;
   1104 }
   1105 
   1106 void dump_emmc_ecsd(const char *ext_csd_path) {
   1107     // List of interesting offsets
   1108     struct hex {
   1109         char str[2];
   1110     };
   1111     static const size_t EXT_CSD_REV = 192 * sizeof(hex);
   1112     static const size_t EXT_PRE_EOL_INFO = 267 * sizeof(hex);
   1113     static const size_t EXT_DEVICE_LIFE_TIME_EST_TYP_A = 268 * sizeof(hex);
   1114     static const size_t EXT_DEVICE_LIFE_TIME_EST_TYP_B = 269 * sizeof(hex);
   1115 
   1116     std::string buffer;
   1117     if (!android::base::ReadFileToString(ext_csd_path, &buffer)) {
   1118         return;
   1119     }
   1120 
   1121     printf("------ %s Extended CSD ------\n", ext_csd_path);
   1122 
   1123     if (buffer.length() < (EXT_CSD_REV + sizeof(hex))) {
   1124         printf("*** %s: truncated content %zu\n\n", ext_csd_path, buffer.length());
   1125         return;
   1126     }
   1127 
   1128     int ext_csd_rev = 0;
   1129     std::string sub = buffer.substr(EXT_CSD_REV, sizeof(hex));
   1130     if (sscanf(sub.c_str(), "%2x", &ext_csd_rev) != 1) {
   1131         printf("*** %s: EXT_CSD_REV parse error \"%s\"\n\n", ext_csd_path, sub.c_str());
   1132         return;
   1133     }
   1134 
   1135     static const char *ver_str[] = {
   1136         "4.0", "4.1", "4.2", "4.3", "Obsolete", "4.41", "4.5", "5.0"
   1137     };
   1138     printf("rev 1.%d (MMC %s)\n", ext_csd_rev,
   1139            (ext_csd_rev < (int)(sizeof(ver_str) / sizeof(ver_str[0]))) ? ver_str[ext_csd_rev]
   1140                                                                        : "Unknown");
   1141     if (ext_csd_rev < 7) {
   1142         printf("\n");
   1143         return;
   1144     }
   1145 
   1146     if (buffer.length() < (EXT_PRE_EOL_INFO + sizeof(hex))) {
   1147         printf("*** %s: truncated content %zu\n\n", ext_csd_path, buffer.length());
   1148         return;
   1149     }
   1150 
   1151     int ext_pre_eol_info = 0;
   1152     sub = buffer.substr(EXT_PRE_EOL_INFO, sizeof(hex));
   1153     if (sscanf(sub.c_str(), "%2x", &ext_pre_eol_info) != 1) {
   1154         printf("*** %s: PRE_EOL_INFO parse error \"%s\"\n\n", ext_csd_path, sub.c_str());
   1155         return;
   1156     }
   1157 
   1158     static const char *eol_str[] = {
   1159         "Undefined",
   1160         "Normal",
   1161         "Warning (consumed 80% of reserve)",
   1162         "Urgent (consumed 90% of reserve)"
   1163     };
   1164     printf(
   1165         "PRE_EOL_INFO %d (MMC %s)\n", ext_pre_eol_info,
   1166         eol_str[(ext_pre_eol_info < (int)(sizeof(eol_str) / sizeof(eol_str[0]))) ? ext_pre_eol_info
   1167                                                                                  : 0]);
   1168 
   1169     for (size_t lifetime = EXT_DEVICE_LIFE_TIME_EST_TYP_A;
   1170             lifetime <= EXT_DEVICE_LIFE_TIME_EST_TYP_B;
   1171             lifetime += sizeof(hex)) {
   1172         int ext_device_life_time_est;
   1173         static const char *est_str[] = {
   1174             "Undefined",
   1175             "0-10% of device lifetime used",
   1176             "10-20% of device lifetime used",
   1177             "20-30% of device lifetime used",
   1178             "30-40% of device lifetime used",
   1179             "40-50% of device lifetime used",
   1180             "50-60% of device lifetime used",
   1181             "60-70% of device lifetime used",
   1182             "70-80% of device lifetime used",
   1183             "80-90% of device lifetime used",
   1184             "90-100% of device lifetime used",
   1185             "Exceeded the maximum estimated device lifetime",
   1186         };
   1187 
   1188         if (buffer.length() < (lifetime + sizeof(hex))) {
   1189             printf("*** %s: truncated content %zu\n", ext_csd_path, buffer.length());
   1190             break;
   1191         }
   1192 
   1193         ext_device_life_time_est = 0;
   1194         sub = buffer.substr(lifetime, sizeof(hex));
   1195         if (sscanf(sub.c_str(), "%2x", &ext_device_life_time_est) != 1) {
   1196             printf("*** %s: DEVICE_LIFE_TIME_EST_TYP_%c parse error \"%s\"\n", ext_csd_path,
   1197                    (unsigned)((lifetime - EXT_DEVICE_LIFE_TIME_EST_TYP_A) / sizeof(hex)) + 'A',
   1198                    sub.c_str());
   1199             continue;
   1200         }
   1201         printf("DEVICE_LIFE_TIME_EST_TYP_%c %d (MMC %s)\n",
   1202                (unsigned)((lifetime - EXT_DEVICE_LIFE_TIME_EST_TYP_A) / sizeof(hex)) + 'A',
   1203                ext_device_life_time_est,
   1204                est_str[(ext_device_life_time_est < (int)(sizeof(est_str) / sizeof(est_str[0])))
   1205                            ? ext_device_life_time_est
   1206                            : 0]);
   1207     }
   1208 
   1209     printf("\n");
   1210 }
   1211