Home | History | Annotate | Download | only in debuggerd
      1 /*
      2  * Copyright (C) 2012 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #include <stddef.h>
     18 #include <stdbool.h>
     19 #include <stdlib.h>
     20 #include <signal.h>
     21 #include <string.h>
     22 #include <stdio.h>
     23 #include <fcntl.h>
     24 #include <errno.h>
     25 #include <dirent.h>
     26 #include <time.h>
     27 #include <sys/ptrace.h>
     28 #include <sys/stat.h>
     29 
     30 #include <private/android_filesystem_config.h>
     31 
     32 #include <cutils/logger.h>
     33 #include <cutils/properties.h>
     34 
     35 #include <corkscrew/demangle.h>
     36 #include <corkscrew/backtrace.h>
     37 
     38 #include "machine.h"
     39 #include "tombstone.h"
     40 #include "utility.h"
     41 
     42 #define STACK_DEPTH 32
     43 #define STACK_WORDS 16
     44 
     45 #define MAX_TOMBSTONES  10
     46 #define TOMBSTONE_DIR   "/data/tombstones"
     47 
     48 #define typecheck(x,y) {    \
     49     typeof(x) __dummy1;     \
     50     typeof(y) __dummy2;     \
     51     (void)(&__dummy1 == &__dummy2); }
     52 
     53 
     54 static bool signal_has_address(int sig) {
     55     switch (sig) {
     56         case SIGILL:
     57         case SIGFPE:
     58         case SIGSEGV:
     59         case SIGBUS:
     60             return true;
     61         default:
     62             return false;
     63     }
     64 }
     65 
     66 static const char *get_signame(int sig)
     67 {
     68     switch(sig) {
     69     case SIGILL:     return "SIGILL";
     70     case SIGABRT:    return "SIGABRT";
     71     case SIGBUS:     return "SIGBUS";
     72     case SIGFPE:     return "SIGFPE";
     73     case SIGSEGV:    return "SIGSEGV";
     74     case SIGPIPE:    return "SIGPIPE";
     75     case SIGSTKFLT:  return "SIGSTKFLT";
     76     case SIGSTOP:    return "SIGSTOP";
     77     default:         return "?";
     78     }
     79 }
     80 
     81 static const char *get_sigcode(int signo, int code)
     82 {
     83     switch (signo) {
     84     case SIGILL:
     85         switch (code) {
     86         case ILL_ILLOPC: return "ILL_ILLOPC";
     87         case ILL_ILLOPN: return "ILL_ILLOPN";
     88         case ILL_ILLADR: return "ILL_ILLADR";
     89         case ILL_ILLTRP: return "ILL_ILLTRP";
     90         case ILL_PRVOPC: return "ILL_PRVOPC";
     91         case ILL_PRVREG: return "ILL_PRVREG";
     92         case ILL_COPROC: return "ILL_COPROC";
     93         case ILL_BADSTK: return "ILL_BADSTK";
     94         }
     95         break;
     96     case SIGBUS:
     97         switch (code) {
     98         case BUS_ADRALN: return "BUS_ADRALN";
     99         case BUS_ADRERR: return "BUS_ADRERR";
    100         case BUS_OBJERR: return "BUS_OBJERR";
    101         }
    102         break;
    103     case SIGFPE:
    104         switch (code) {
    105         case FPE_INTDIV: return "FPE_INTDIV";
    106         case FPE_INTOVF: return "FPE_INTOVF";
    107         case FPE_FLTDIV: return "FPE_FLTDIV";
    108         case FPE_FLTOVF: return "FPE_FLTOVF";
    109         case FPE_FLTUND: return "FPE_FLTUND";
    110         case FPE_FLTRES: return "FPE_FLTRES";
    111         case FPE_FLTINV: return "FPE_FLTINV";
    112         case FPE_FLTSUB: return "FPE_FLTSUB";
    113         }
    114         break;
    115     case SIGSEGV:
    116         switch (code) {
    117         case SEGV_MAPERR: return "SEGV_MAPERR";
    118         case SEGV_ACCERR: return "SEGV_ACCERR";
    119         }
    120         break;
    121     }
    122     return "?";
    123 }
    124 
    125 static void dump_build_info(log_t* log)
    126 {
    127     char fingerprint[PROPERTY_VALUE_MAX];
    128 
    129     property_get("ro.build.fingerprint", fingerprint, "unknown");
    130 
    131     _LOG(log, false, "Build fingerprint: '%s'\n", fingerprint);
    132 }
    133 
    134 static void dump_fault_addr(log_t* log, pid_t tid, int sig)
    135 {
    136     siginfo_t si;
    137 
    138     memset(&si, 0, sizeof(si));
    139     if(ptrace(PTRACE_GETSIGINFO, tid, 0, &si)){
    140         _LOG(log, false, "cannot get siginfo: %s\n", strerror(errno));
    141     } else if (signal_has_address(sig)) {
    142         _LOG(log, false, "signal %d (%s), code %d (%s), fault addr %08x\n",
    143              sig, get_signame(sig),
    144              si.si_code, get_sigcode(sig, si.si_code),
    145              (uintptr_t) si.si_addr);
    146     } else {
    147         _LOG(log, false, "signal %d (%s), code %d (%s), fault addr --------\n",
    148              sig, get_signame(sig), si.si_code, get_sigcode(sig, si.si_code));
    149     }
    150 }
    151 
    152 static void dump_thread_info(log_t* log, pid_t pid, pid_t tid, bool at_fault) {
    153     char path[64];
    154     char threadnamebuf[1024];
    155     char* threadname = NULL;
    156     FILE *fp;
    157 
    158     snprintf(path, sizeof(path), "/proc/%d/comm", tid);
    159     if ((fp = fopen(path, "r"))) {
    160         threadname = fgets(threadnamebuf, sizeof(threadnamebuf), fp);
    161         fclose(fp);
    162         if (threadname) {
    163             size_t len = strlen(threadname);
    164             if (len && threadname[len - 1] == '\n') {
    165                 threadname[len - 1] = '\0';
    166             }
    167         }
    168     }
    169 
    170     if (at_fault) {
    171         char procnamebuf[1024];
    172         char* procname = NULL;
    173 
    174         snprintf(path, sizeof(path), "/proc/%d/cmdline", pid);
    175         if ((fp = fopen(path, "r"))) {
    176             procname = fgets(procnamebuf, sizeof(procnamebuf), fp);
    177             fclose(fp);
    178         }
    179 
    180         _LOG(log, false, "pid: %d, tid: %d, name: %s  >>> %s <<<\n", pid, tid,
    181                 threadname ? threadname : "UNKNOWN",
    182                 procname ? procname : "UNKNOWN");
    183     } else {
    184         _LOG(log, true, "pid: %d, tid: %d, name: %s\n", pid, tid,
    185                 threadname ? threadname : "UNKNOWN");
    186     }
    187 }
    188 
    189 static void dump_backtrace(const ptrace_context_t* context __attribute((unused)),
    190         log_t* log, pid_t tid __attribute((unused)), bool at_fault,
    191         const backtrace_frame_t* backtrace, size_t frames) {
    192     _LOG(log, !at_fault, "\nbacktrace:\n");
    193 
    194     backtrace_symbol_t backtrace_symbols[STACK_DEPTH];
    195     get_backtrace_symbols_ptrace(context, backtrace, frames, backtrace_symbols);
    196     for (size_t i = 0; i < frames; i++) {
    197         char line[MAX_BACKTRACE_LINE_LENGTH];
    198         format_backtrace_line(i, &backtrace[i], &backtrace_symbols[i],
    199                 line, MAX_BACKTRACE_LINE_LENGTH);
    200         _LOG(log, !at_fault, "    %s\n", line);
    201     }
    202     free_backtrace_symbols(backtrace_symbols, frames);
    203 }
    204 
    205 static void dump_stack_segment(const ptrace_context_t* context, log_t* log, pid_t tid,
    206         bool only_in_tombstone, uintptr_t* sp, size_t words, int label) {
    207     for (size_t i = 0; i < words; i++) {
    208         uint32_t stack_content;
    209         if (!try_get_word_ptrace(tid, *sp, &stack_content)) {
    210             break;
    211         }
    212 
    213         const map_info_t* mi;
    214         const symbol_t* symbol;
    215         find_symbol_ptrace(context, stack_content, &mi, &symbol);
    216 
    217         if (symbol) {
    218             char* demangled_name = demangle_symbol_name(symbol->name);
    219             const char* symbol_name = demangled_name ? demangled_name : symbol->name;
    220             uint32_t offset = stack_content - (mi->start + symbol->start);
    221             if (!i && label >= 0) {
    222                 if (offset) {
    223                     _LOG(log, only_in_tombstone, "    #%02d  %08x  %08x  %s (%s+%u)\n",
    224                             label, *sp, stack_content, mi ? mi->name : "", symbol_name, offset);
    225                 } else {
    226                     _LOG(log, only_in_tombstone, "    #%02d  %08x  %08x  %s (%s)\n",
    227                             label, *sp, stack_content, mi ? mi->name : "", symbol_name);
    228                 }
    229             } else {
    230                 if (offset) {
    231                     _LOG(log, only_in_tombstone, "         %08x  %08x  %s (%s+%u)\n",
    232                             *sp, stack_content, mi ? mi->name : "", symbol_name, offset);
    233                 } else {
    234                     _LOG(log, only_in_tombstone, "         %08x  %08x  %s (%s)\n",
    235                             *sp, stack_content, mi ? mi->name : "", symbol_name);
    236                 }
    237             }
    238             free(demangled_name);
    239         } else {
    240             if (!i && label >= 0) {
    241                 _LOG(log, only_in_tombstone, "    #%02d  %08x  %08x  %s\n",
    242                         label, *sp, stack_content, mi ? mi->name : "");
    243             } else {
    244                 _LOG(log, only_in_tombstone, "         %08x  %08x  %s\n",
    245                         *sp, stack_content, mi ? mi->name : "");
    246             }
    247         }
    248 
    249         *sp += sizeof(uint32_t);
    250     }
    251 }
    252 
    253 static void dump_stack(const ptrace_context_t* context, log_t* log, pid_t tid, bool at_fault,
    254         const backtrace_frame_t* backtrace, size_t frames) {
    255     bool have_first = false;
    256     size_t first, last;
    257     for (size_t i = 0; i < frames; i++) {
    258         if (backtrace[i].stack_top) {
    259             if (!have_first) {
    260                 have_first = true;
    261                 first = i;
    262             }
    263             last = i;
    264         }
    265     }
    266     if (!have_first) {
    267         return;
    268     }
    269 
    270     _LOG(log, !at_fault, "\nstack:\n");
    271 
    272     // Dump a few words before the first frame.
    273     bool only_in_tombstone = !at_fault;
    274     uintptr_t sp = backtrace[first].stack_top - STACK_WORDS * sizeof(uint32_t);
    275     dump_stack_segment(context, log, tid, only_in_tombstone, &sp, STACK_WORDS, -1);
    276 
    277     // Dump a few words from all successive frames.
    278     // Only log the first 3 frames, put the rest in the tombstone.
    279     for (size_t i = first; i <= last; i++) {
    280         const backtrace_frame_t* frame = &backtrace[i];
    281         if (sp != frame->stack_top) {
    282             _LOG(log, only_in_tombstone, "         ........  ........\n");
    283             sp = frame->stack_top;
    284         }
    285         if (i - first == 3) {
    286             only_in_tombstone = true;
    287         }
    288         if (i == last) {
    289             dump_stack_segment(context, log, tid, only_in_tombstone, &sp, STACK_WORDS, i);
    290             if (sp < frame->stack_top + frame->stack_size) {
    291                 _LOG(log, only_in_tombstone, "         ........  ........\n");
    292             }
    293         } else {
    294             size_t words = frame->stack_size / sizeof(uint32_t);
    295             if (words == 0) {
    296                 words = 1;
    297             } else if (words > STACK_WORDS) {
    298                 words = STACK_WORDS;
    299             }
    300             dump_stack_segment(context, log, tid, only_in_tombstone, &sp, words, i);
    301         }
    302     }
    303 }
    304 
    305 static void dump_backtrace_and_stack(const ptrace_context_t* context, log_t* log, pid_t tid,
    306         bool at_fault) {
    307     backtrace_frame_t backtrace[STACK_DEPTH];
    308     ssize_t frames = unwind_backtrace_ptrace(tid, context, backtrace, 0, STACK_DEPTH);
    309     if (frames > 0) {
    310         dump_backtrace(context, log, tid, at_fault, backtrace, frames);
    311         dump_stack(context, log, tid, at_fault, backtrace, frames);
    312     }
    313 }
    314 
    315 static void dump_nearby_maps(const ptrace_context_t* context, log_t* log, pid_t tid) {
    316     siginfo_t si;
    317     memset(&si, 0, sizeof(si));
    318     if (ptrace(PTRACE_GETSIGINFO, tid, 0, &si)) {
    319         _LOG(log, false, "cannot get siginfo for %d: %s\n",
    320                 tid, strerror(errno));
    321         return;
    322     }
    323     if (!signal_has_address(si.si_signo)) {
    324         return;
    325     }
    326 
    327     uintptr_t addr = (uintptr_t) si.si_addr;
    328     addr &= ~0xfff;     /* round to 4K page boundary */
    329     if (addr == 0) {    /* null-pointer deref */
    330         return;
    331     }
    332 
    333     _LOG(log, false, "\nmemory map around fault addr %08x:\n", (int)si.si_addr);
    334 
    335     /*
    336      * Search for a match, or for a hole where the match would be.  The list
    337      * is backward from the file content, so it starts at high addresses.
    338      */
    339     bool found = false;
    340     map_info_t* map = context->map_info_list;
    341     map_info_t *next = NULL;
    342     map_info_t *prev = NULL;
    343     while (map != NULL) {
    344         if (addr >= map->start && addr < map->end) {
    345             found = true;
    346             next = map->next;
    347             break;
    348         } else if (addr >= map->end) {
    349             /* map would be between "prev" and this entry */
    350             next = map;
    351             map = NULL;
    352             break;
    353         }
    354 
    355         prev = map;
    356         map = map->next;
    357     }
    358 
    359     /*
    360      * Show "next" then "match" then "prev" so that the addresses appear in
    361      * ascending order (like /proc/pid/maps).
    362      */
    363     if (next != NULL) {
    364         _LOG(log, false, "    %08x-%08x %s\n", next->start, next->end, next->name);
    365     } else {
    366         _LOG(log, false, "    (no map below)\n");
    367     }
    368     if (map != NULL) {
    369         _LOG(log, false, "    %08x-%08x %s\n", map->start, map->end, map->name);
    370     } else {
    371         _LOG(log, false, "    (no map for address)\n");
    372     }
    373     if (prev != NULL) {
    374         _LOG(log, false, "    %08x-%08x %s\n", prev->start, prev->end, prev->name);
    375     } else {
    376         _LOG(log, false, "    (no map above)\n");
    377     }
    378 }
    379 
    380 static void dump_thread(const ptrace_context_t* context, log_t* log, pid_t tid, bool at_fault,
    381         int* total_sleep_time_usec) {
    382     wait_for_stop(tid, total_sleep_time_usec);
    383 
    384     dump_registers(context, log, tid, at_fault);
    385     dump_backtrace_and_stack(context, log, tid, at_fault);
    386     if (at_fault) {
    387         dump_memory_and_code(context, log, tid, at_fault);
    388         dump_nearby_maps(context, log, tid);
    389     }
    390 }
    391 
    392 /* Return true if some thread is not detached cleanly */
    393 static bool dump_sibling_thread_report(const ptrace_context_t* context,
    394         log_t* log, pid_t pid, pid_t tid, int* total_sleep_time_usec) {
    395     char task_path[64];
    396     snprintf(task_path, sizeof(task_path), "/proc/%d/task", pid);
    397 
    398     DIR* d = opendir(task_path);
    399     /* Bail early if cannot open the task directory */
    400     if (d == NULL) {
    401         XLOG("Cannot open /proc/%d/task\n", pid);
    402         return false;
    403     }
    404 
    405     bool detach_failed = false;
    406     struct dirent debuf;
    407     struct dirent *de;
    408     while (!readdir_r(d, &debuf, &de) && de) {
    409         /* Ignore "." and ".." */
    410         if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) {
    411             continue;
    412         }
    413 
    414         /* The main thread at fault has been handled individually */
    415         char* end;
    416         pid_t new_tid = strtoul(de->d_name, &end, 10);
    417         if (*end || new_tid == tid) {
    418             continue;
    419         }
    420 
    421         /* Skip this thread if cannot ptrace it */
    422         if (ptrace(PTRACE_ATTACH, new_tid, 0, 0) < 0) {
    423             continue;
    424         }
    425 
    426         _LOG(log, true, "--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---\n");
    427         dump_thread_info(log, pid, new_tid, false);
    428         dump_thread(context, log, new_tid, false, total_sleep_time_usec);
    429 
    430         if (ptrace(PTRACE_DETACH, new_tid, 0, 0) != 0) {
    431             LOG("ptrace detach from %d failed: %s\n", new_tid, strerror(errno));
    432             detach_failed = true;
    433         }
    434     }
    435 
    436     closedir(d);
    437     return detach_failed;
    438 }
    439 
    440 /*
    441  * Reads the contents of the specified log device, filters out the entries
    442  * that don't match the specified pid, and writes them to the tombstone file.
    443  *
    444  * If "tailOnly" is set, we only print the last few lines.
    445  */
    446 static void dump_log_file(log_t* log, pid_t pid, const char* filename,
    447     bool tailOnly)
    448 {
    449     bool first = true;
    450 
    451     /* circular buffer, for "tailOnly" mode */
    452     const int kShortLogMaxLines = 5;
    453     const int kShortLogLineLen = 256;
    454     char shortLog[kShortLogMaxLines][kShortLogLineLen];
    455     int shortLogCount = 0;
    456     int shortLogNext = 0;
    457 
    458     int logfd = open(filename, O_RDONLY | O_NONBLOCK);
    459     if (logfd < 0) {
    460         XLOG("Unable to open %s: %s\n", filename, strerror(errno));
    461         return;
    462     }
    463 
    464     union {
    465         unsigned char buf[LOGGER_ENTRY_MAX_LEN + 1];
    466         struct logger_entry entry;
    467     } log_entry;
    468 
    469     while (true) {
    470         ssize_t actual = read(logfd, log_entry.buf, LOGGER_ENTRY_MAX_LEN);
    471         if (actual < 0) {
    472             if (errno == EINTR) {
    473                 /* interrupted by signal, retry */
    474                 continue;
    475             } else if (errno == EAGAIN) {
    476                 /* non-blocking EOF; we're done */
    477                 break;
    478             } else {
    479                 _LOG(log, true, "Error while reading log: %s\n",
    480                     strerror(errno));
    481                 break;
    482             }
    483         } else if (actual == 0) {
    484             _LOG(log, true, "Got zero bytes while reading log: %s\n",
    485                 strerror(errno));
    486             break;
    487         }
    488 
    489         /*
    490          * NOTE: if you XLOG something here, this will spin forever,
    491          * because you will be writing as fast as you're reading.  Any
    492          * high-frequency debug diagnostics should just be written to
    493          * the tombstone file.
    494          */
    495 
    496         struct logger_entry* entry = &log_entry.entry;
    497 
    498         if (entry->pid != (int32_t) pid) {
    499             /* wrong pid, ignore */
    500             continue;
    501         }
    502 
    503         if (first) {
    504             _LOG(log, true, "--------- %slog %s\n",
    505                 tailOnly ? "tail end of " : "", filename);
    506             first = false;
    507         }
    508 
    509         /*
    510          * Msg format is: <priority:1><tag:N>\0<message:N>\0
    511          *
    512          * We want to display it in the same format as "logcat -v threadtime"
    513          * (although in this case the pid is redundant).
    514          *
    515          * TODO: scan for line breaks ('\n') and display each text line
    516          * on a separate line, prefixed with the header, like logcat does.
    517          */
    518         static const char* kPrioChars = "!.VDIWEFS";
    519         unsigned char prio = entry->msg[0];
    520         char* tag = entry->msg + 1;
    521         char* msg = tag + strlen(tag) + 1;
    522 
    523         /* consume any trailing newlines */
    524         char* eatnl = msg + strlen(msg) - 1;
    525         while (eatnl >= msg && *eatnl == '\n') {
    526             *eatnl-- = '\0';
    527         }
    528 
    529         char prioChar = (prio < strlen(kPrioChars) ? kPrioChars[prio] : '?');
    530 
    531         char timeBuf[32];
    532         time_t sec = (time_t) entry->sec;
    533         struct tm tmBuf;
    534         struct tm* ptm;
    535         ptm = localtime_r(&sec, &tmBuf);
    536         strftime(timeBuf, sizeof(timeBuf), "%m-%d %H:%M:%S", ptm);
    537 
    538         if (tailOnly) {
    539             snprintf(shortLog[shortLogNext], kShortLogLineLen,
    540                 "%s.%03d %5d %5d %c %-8s: %s",
    541                 timeBuf, entry->nsec / 1000000, entry->pid, entry->tid,
    542                 prioChar, tag, msg);
    543             shortLogNext = (shortLogNext + 1) % kShortLogMaxLines;
    544             shortLogCount++;
    545         } else {
    546             _LOG(log, true, "%s.%03d %5d %5d %c %-8s: %s\n",
    547                 timeBuf, entry->nsec / 1000000, entry->pid, entry->tid,
    548                 prioChar, tag, msg);
    549         }
    550     }
    551 
    552     if (tailOnly) {
    553         int i;
    554 
    555         /*
    556          * If we filled the buffer, we want to start at "next", which has
    557          * the oldest entry.  If we didn't, we want to start at zero.
    558          */
    559         if (shortLogCount < kShortLogMaxLines) {
    560             shortLogNext = 0;
    561         } else {
    562             shortLogCount = kShortLogMaxLines;  /* cap at window size */
    563         }
    564 
    565         for (i = 0; i < shortLogCount; i++) {
    566             _LOG(log, true, "%s\n", shortLog[shortLogNext]);
    567             shortLogNext = (shortLogNext + 1) % kShortLogMaxLines;
    568         }
    569     }
    570 
    571     close(logfd);
    572 }
    573 
    574 /*
    575  * Dumps the logs generated by the specified pid to the tombstone, from both
    576  * "system" and "main" log devices.  Ideally we'd interleave the output.
    577  */
    578 static void dump_logs(log_t* log, pid_t pid, bool tailOnly)
    579 {
    580     dump_log_file(log, pid, "/dev/log/system", tailOnly);
    581     dump_log_file(log, pid, "/dev/log/main", tailOnly);
    582 }
    583 
    584 /*
    585  * Dumps all information about the specified pid to the tombstone.
    586  */
    587 static bool dump_crash(log_t* log, pid_t pid, pid_t tid, int signal,
    588         bool dump_sibling_threads, int* total_sleep_time_usec)
    589 {
    590     /* don't copy log messages to tombstone unless this is a dev device */
    591     char value[PROPERTY_VALUE_MAX];
    592     property_get("ro.debuggable", value, "0");
    593     bool want_logs = (value[0] == '1');
    594 
    595     _LOG(log, false,
    596             "*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***\n");
    597     dump_build_info(log);
    598     dump_thread_info(log, pid, tid, true);
    599     if(signal) {
    600         dump_fault_addr(log, tid, signal);
    601     }
    602 
    603     ptrace_context_t* context = load_ptrace_context(tid);
    604     dump_thread(context, log, tid, true, total_sleep_time_usec);
    605 
    606     if (want_logs) {
    607         dump_logs(log, pid, true);
    608     }
    609 
    610     bool detach_failed = false;
    611     if (dump_sibling_threads) {
    612         detach_failed = dump_sibling_thread_report(context, log, pid, tid, total_sleep_time_usec);
    613     }
    614 
    615     free_ptrace_context(context);
    616 
    617     if (want_logs) {
    618         dump_logs(log, pid, false);
    619     }
    620     return detach_failed;
    621 }
    622 
    623 /*
    624  * find_and_open_tombstone - find an available tombstone slot, if any, of the
    625  * form tombstone_XX where XX is 00 to MAX_TOMBSTONES-1, inclusive. If no
    626  * file is available, we reuse the least-recently-modified file.
    627  *
    628  * Returns the path of the tombstone file, allocated using malloc().  Caller must free() it.
    629  */
    630 static char* find_and_open_tombstone(int* fd)
    631 {
    632     unsigned long mtime = ULONG_MAX;
    633     struct stat sb;
    634 
    635     /*
    636      * XXX: Our stat.st_mtime isn't time_t. If it changes, as it probably ought
    637      * to, our logic breaks. This check will generate a warning if that happens.
    638      */
    639     typecheck(mtime, sb.st_mtime);
    640 
    641     /*
    642      * In a single wolf-like pass, find an available slot and, in case none
    643      * exist, find and record the least-recently-modified file.
    644      */
    645     char path[128];
    646     int oldest = 0;
    647     for (int i = 0; i < MAX_TOMBSTONES; i++) {
    648         snprintf(path, sizeof(path), TOMBSTONE_DIR"/tombstone_%02d", i);
    649 
    650         if (!stat(path, &sb)) {
    651             if (sb.st_mtime < mtime) {
    652                 oldest = i;
    653                 mtime = sb.st_mtime;
    654             }
    655             continue;
    656         }
    657         if (errno != ENOENT)
    658             continue;
    659 
    660         *fd = open(path, O_CREAT | O_EXCL | O_WRONLY, 0600);
    661         if (*fd < 0)
    662             continue;   /* raced ? */
    663 
    664         fchown(*fd, AID_SYSTEM, AID_SYSTEM);
    665         return strdup(path);
    666     }
    667 
    668     /* we didn't find an available file, so we clobber the oldest one */
    669     snprintf(path, sizeof(path), TOMBSTONE_DIR"/tombstone_%02d", oldest);
    670     *fd = open(path, O_CREAT | O_TRUNC | O_WRONLY, 0600);
    671     if (*fd < 0) {
    672         LOG("failed to open tombstone file '%s': %s\n", path, strerror(errno));
    673         return NULL;
    674     }
    675     fchown(*fd, AID_SYSTEM, AID_SYSTEM);
    676     return strdup(path);
    677 }
    678 
    679 char* engrave_tombstone(pid_t pid, pid_t tid, int signal,
    680         bool dump_sibling_threads, bool quiet, bool* detach_failed,
    681         int* total_sleep_time_usec) {
    682     mkdir(TOMBSTONE_DIR, 0755);
    683     chown(TOMBSTONE_DIR, AID_SYSTEM, AID_SYSTEM);
    684 
    685     int fd;
    686     char* path = find_and_open_tombstone(&fd);
    687     if (!path) {
    688         *detach_failed = false;
    689         return NULL;
    690     }
    691 
    692     log_t log;
    693     log.tfd = fd;
    694     log.quiet = quiet;
    695     *detach_failed = dump_crash(&log, pid, tid, signal, dump_sibling_threads,
    696             total_sleep_time_usec);
    697 
    698     close(fd);
    699     return path;
    700 }
    701