Home | History | Annotate | Download | only in src
      1 // Copyright (c) 2005, Google Inc.
      2 // All rights reserved.
      3 //
      4 // Redistribution and use in source and binary forms, with or without
      5 // modification, are permitted provided that the following conditions are
      6 // met:
      7 //
      8 //     * Redistributions of source code must retain the above copyright
      9 // notice, this list of conditions and the following disclaimer.
     10 //     * Redistributions in binary form must reproduce the above
     11 // copyright notice, this list of conditions and the following disclaimer
     12 // in the documentation and/or other materials provided with the
     13 // distribution.
     14 //     * Neither the name of Google Inc. nor the names of its
     15 // contributors may be used to endorse or promote products derived from
     16 // this software without specific prior written permission.
     17 //
     18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     29 
     30 // ---
     31 // All Rights Reserved.
     32 //
     33 // Author: Maxim Lifantsev
     34 //
     35 
     36 #include "config.h"
     37 
     38 #include <fcntl.h>    // for O_RDONLY (we use syscall to do actual reads)
     39 #include <string.h>
     40 #include <errno.h>
     41 #ifdef HAVE_UNISTD_H
     42 #include <unistd.h>
     43 #endif
     44 #ifdef HAVE_MMAP
     45 #include <sys/mman.h>
     46 #endif
     47 #ifdef HAVE_PTHREAD
     48 #include <pthread.h>
     49 #endif
     50 #include <sys/stat.h>
     51 #include <sys/types.h>
     52 #include <time.h>
     53 #include <assert.h>
     54 
     55 #if defined(HAVE_LINUX_PTRACE_H)
     56 #include <linux/ptrace.h>
     57 #endif
     58 #ifdef HAVE_SYS_SYSCALL_H
     59 #include <sys/syscall.h>
     60 #endif
     61 #if defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__MINGW32__)
     62 #include <wtypes.h>
     63 #include <winbase.h>
     64 #undef ERROR     // windows defines these as macros, which can cause trouble
     65 #undef max
     66 #undef min
     67 #endif
     68 
     69 #include <string>
     70 #include <vector>
     71 #include <map>
     72 #include <set>
     73 #include <algorithm>
     74 #include <functional>
     75 
     76 #include <gperftools/heap-checker.h>
     77 
     78 #include "base/basictypes.h"
     79 #include "base/googleinit.h"
     80 #include "base/logging.h"
     81 #include <gperftools/stacktrace.h>
     82 #include "base/commandlineflags.h"
     83 #include "base/elfcore.h"              // for i386_regs
     84 #include "base/thread_lister.h"
     85 #include "heap-profile-table.h"
     86 #include "base/low_level_alloc.h"
     87 #include "malloc_hook-inl.h"
     88 #include <gperftools/malloc_hook.h>
     89 #include <gperftools/malloc_extension.h>
     90 #include "maybe_threads.h"
     91 #include "memory_region_map.h"
     92 #include "base/spinlock.h"
     93 #include "base/sysinfo.h"
     94 #include "base/stl_allocator.h"
     95 
     96 using std::string;
     97 using std::basic_string;
     98 using std::pair;
     99 using std::map;
    100 using std::set;
    101 using std::vector;
    102 using std::swap;
    103 using std::make_pair;
    104 using std::min;
    105 using std::max;
    106 using std::less;
    107 using std::char_traits;
    108 
    109 // If current process is being ptrace()d, 'TracerPid' in /proc/self/status
    110 // will be non-zero.
    111 static bool IsDebuggerAttached(void) {    // only works under linux, probably
    112   char buf[256];   // TracerPid comes relatively earlier in status output
    113   int fd = open("/proc/self/status", O_RDONLY);
    114   if (fd == -1) {
    115     return false;  // Can't tell for sure.
    116   }
    117   const int len = read(fd, buf, sizeof(buf));
    118   bool rc = false;
    119   if (len > 0) {
    120     const char *const kTracerPid = "TracerPid:\t";
    121     buf[len - 1] = '\0';
    122     const char *p = strstr(buf, kTracerPid);
    123     if (p != NULL) {
    124       rc = (strncmp(p + strlen(kTracerPid), "0\n", 2) != 0);
    125     }
    126   }
    127   close(fd);
    128   return rc;
    129 }
    130 
    131 // This is the default if you don't link in -lprofiler
    132 extern "C" {
    133 ATTRIBUTE_WEAK PERFTOOLS_DLL_DECL bool ProfilingIsEnabledForAllThreads();
    134 bool ProfilingIsEnabledForAllThreads() { return false; }
    135 }
    136 
    137 //----------------------------------------------------------------------
    138 // Flags that control heap-checking
    139 //----------------------------------------------------------------------
    140 
    141 DEFINE_string(heap_check,
    142               EnvToString("HEAPCHECK", ""),
    143               "The heap leak checking to be done over the whole executable: "
    144               "\"minimal\", \"normal\", \"strict\", "
    145               "\"draconian\", \"as-is\", and \"local\" "
    146               " or the empty string are the supported choices. "
    147               "(See HeapLeakChecker_InternalInitStart for details.)");
    148 
    149 DEFINE_bool(heap_check_report, true, "Obsolete");
    150 
    151 DEFINE_bool(heap_check_before_constructors,
    152             true,
    153             "deprecated; pretty much always true now");
    154 
    155 DEFINE_bool(heap_check_after_destructors,
    156             EnvToBool("HEAP_CHECK_AFTER_DESTRUCTORS", false),
    157             "If overall heap check is to end after global destructors "
    158             "or right after all REGISTER_HEAPCHECK_CLEANUP's");
    159 
    160 DEFINE_bool(heap_check_strict_check, true, "Obsolete");
    161 
    162 DEFINE_bool(heap_check_ignore_global_live,
    163             EnvToBool("HEAP_CHECK_IGNORE_GLOBAL_LIVE", true),
    164             "If overall heap check is to ignore heap objects reachable "
    165             "from the global data");
    166 
    167 DEFINE_bool(heap_check_identify_leaks,
    168             EnvToBool("HEAP_CHECK_IDENTIFY_LEAKS", false),
    169             "If heap check should generate the addresses of the leaked "
    170             "objects in the memory leak profiles.  This may be useful "
    171             "in tracking down leaks where only a small fraction of "
    172             "objects allocated at the same stack trace are leaked.");
    173 
    174 DEFINE_bool(heap_check_ignore_thread_live,
    175             EnvToBool("HEAP_CHECK_IGNORE_THREAD_LIVE", true),
    176             "If set to true, objects reachable from thread stacks "
    177             "and registers are not reported as leaks");
    178 
    179 DEFINE_bool(heap_check_test_pointer_alignment,
    180             EnvToBool("HEAP_CHECK_TEST_POINTER_ALIGNMENT", false),
    181             "Set to true to check if the found leak can be due to "
    182             "use of unaligned pointers");
    183 
    184 // Alignment at which all pointers in memory are supposed to be located;
    185 // use 1 if any alignment is ok.
    186 // heap_check_test_pointer_alignment flag guides if we try the value of 1.
    187 // The larger it can be, the lesser is the chance of missing real leaks.
    188 static const size_t kPointerSourceAlignment = sizeof(void*);
    189 DEFINE_int32(heap_check_pointer_source_alignment,
    190 	     EnvToInt("HEAP_CHECK_POINTER_SOURCE_ALIGNMENT",
    191                       kPointerSourceAlignment),
    192              "Alignment at which all pointers in memory are supposed to be "
    193              "located.  Use 1 if any alignment is ok.");
    194 
    195 // A reasonable default to handle pointers inside of typical class objects:
    196 // Too low and we won't be able to traverse pointers to normally-used
    197 // nested objects and base parts of multiple-inherited objects.
    198 // Too high and it will both slow down leak checking (FindInsideAlloc
    199 // in HaveOnHeapLocked will get slower when there are large on-heap objects)
    200 // and make it probabilistically more likely to miss leaks
    201 // of large-sized objects.
    202 static const int64 kHeapCheckMaxPointerOffset = 1024;
    203 DEFINE_int64(heap_check_max_pointer_offset,
    204 	     EnvToInt("HEAP_CHECK_MAX_POINTER_OFFSET",
    205                       kHeapCheckMaxPointerOffset),
    206              "Largest pointer offset for which we traverse "
    207              "pointers going inside of heap allocated objects. "
    208              "Set to -1 to use the actual largest heap object size.");
    209 
    210 DEFINE_bool(heap_check_run_under_gdb,
    211             EnvToBool("HEAP_CHECK_RUN_UNDER_GDB", false),
    212             "If false, turns off heap-checking library when running under gdb "
    213             "(normally, set to 'true' only when debugging the heap-checker)");
    214 
    215 DEFINE_int32(heap_check_delay_seconds, 0,
    216              "Number of seconds to delay on-exit heap checking."
    217              " If you set this flag,"
    218              " you may also want to set exit_timeout_seconds in order to"
    219              " avoid exit timeouts.\n"
    220              "NOTE: This flag is to be used only to help diagnose issues"
    221              " where it is suspected that the heap checker is reporting"
    222              " false leaks that will disappear if the heap checker delays"
    223              " its checks. Report any such issues to the heap-checker"
    224              " maintainer(s).");
    225 
    226 DEFINE_int32(heap_check_error_exit_code,
    227              EnvToInt("HEAP_CHECK_ERROR_EXIT_CODE", 1),
    228              "Exit code to return if any leaks were detected.");
    229 
    230 //----------------------------------------------------------------------
    231 
    232 DEFINE_string(heap_profile_pprof,
    233               EnvToString("PPROF_PATH", "pprof"),
    234               "OBSOLETE; not used");
    235 
    236 DEFINE_string(heap_check_dump_directory,
    237               EnvToString("HEAP_CHECK_DUMP_DIRECTORY", "/tmp"),
    238               "Directory to put heap-checker leak dump information");
    239 
    240 
    241 //----------------------------------------------------------------------
    242 // HeapLeakChecker global data
    243 //----------------------------------------------------------------------
    244 
    245 // Global lock for all the global data of this module.
    246 static SpinLock heap_checker_lock(SpinLock::LINKER_INITIALIZED);
    247 
    248 //----------------------------------------------------------------------
    249 
    250 // Heap profile prefix for leak checking profiles.
    251 // Gets assigned once when leak checking is turned on, then never modified.
    252 static const string* profile_name_prefix = NULL;
    253 
    254 // Whole-program heap leak checker.
    255 // Gets assigned once when leak checking is turned on,
    256 // then main_heap_checker is never deleted.
    257 static HeapLeakChecker* main_heap_checker = NULL;
    258 
    259 // Whether we will use main_heap_checker to do a check at program exit
    260 // automatically. In any case user can ask for more checks on main_heap_checker
    261 // via GlobalChecker().
    262 static bool do_main_heap_check = false;
    263 
    264 // The heap profile we use to collect info about the heap.
    265 // This is created in HeapLeakChecker::BeforeConstructorsLocked
    266 // together with setting heap_checker_on (below) to true
    267 // and registering our new/delete malloc hooks;
    268 // similarly all are unset in HeapLeakChecker::TurnItselfOffLocked.
    269 static HeapProfileTable* heap_profile = NULL;
    270 
    271 // If we are doing (or going to do) any kind of heap-checking.
    272 static bool heap_checker_on = false;
    273 
    274 // pid of the process that does whole-program heap leak checking
    275 static pid_t heap_checker_pid = 0;
    276 
    277 // If we did heap profiling during global constructors execution
    278 static bool constructor_heap_profiling = false;
    279 
    280 // RAW_VLOG level we dump key INFO messages at.  If you want to turn
    281 // off these messages, set the environment variable PERFTOOLS_VERBOSE=-1.
    282 static const int heap_checker_info_level = 0;
    283 
    284 //----------------------------------------------------------------------
    285 // HeapLeakChecker's own memory allocator that is
    286 // independent of the normal program allocator.
    287 //----------------------------------------------------------------------
    288 
    289 // Wrapper of LowLevelAlloc for STL_Allocator and direct use.
    290 // We always access this class under held heap_checker_lock,
    291 // this allows us to in particular protect the period when threads are stopped
    292 // at random spots with ListAllProcessThreads by heap_checker_lock,
    293 // w/o worrying about the lock in LowLevelAlloc::Arena.
    294 // We rely on the fact that we use an own arena with an own lock here.
    295 class HeapLeakChecker::Allocator {
    296  public:
    297   static void Init() {
    298     RAW_DCHECK(heap_checker_lock.IsHeld(), "");
    299     RAW_DCHECK(arena_ == NULL, "");
    300     arena_ = LowLevelAlloc::NewArena(0, LowLevelAlloc::DefaultArena());
    301   }
    302   static void Shutdown() {
    303     RAW_DCHECK(heap_checker_lock.IsHeld(), "");
    304     if (!LowLevelAlloc::DeleteArena(arena_)  ||  alloc_count_ != 0) {
    305       RAW_LOG(FATAL, "Internal heap checker leak of %d objects", alloc_count_);
    306     }
    307   }
    308   static int alloc_count() {
    309     RAW_DCHECK(heap_checker_lock.IsHeld(), "");
    310     return alloc_count_;
    311   }
    312   static void* Allocate(size_t n) {
    313     RAW_DCHECK(arena_  &&  heap_checker_lock.IsHeld(), "");
    314     void* p = LowLevelAlloc::AllocWithArena(n, arena_);
    315     if (p) alloc_count_ += 1;
    316     return p;
    317   }
    318   static void Free(void* p) {
    319     RAW_DCHECK(heap_checker_lock.IsHeld(), "");
    320     if (p) alloc_count_ -= 1;
    321     LowLevelAlloc::Free(p);
    322   }
    323   static void Free(void* p, size_t /* n */) {
    324     Free(p);
    325   }
    326   // destruct, free, and make *p to be NULL
    327   template<typename T> static void DeleteAndNull(T** p) {
    328     (*p)->~T();
    329     Free(*p);
    330     *p = NULL;
    331   }
    332   template<typename T> static void DeleteAndNullIfNot(T** p) {
    333     if (*p != NULL) DeleteAndNull(p);
    334   }
    335  private:
    336   static LowLevelAlloc::Arena* arena_;
    337   static int alloc_count_;
    338 };
    339 
    340 LowLevelAlloc::Arena* HeapLeakChecker::Allocator::arena_ = NULL;
    341 int HeapLeakChecker::Allocator::alloc_count_ = 0;
    342 
    343 //----------------------------------------------------------------------
    344 // HeapLeakChecker live object tracking components
    345 //----------------------------------------------------------------------
    346 
    347 // Cases of live object placement we distinguish
    348 enum ObjectPlacement {
    349   MUST_BE_ON_HEAP,   // Must point to a live object of the matching size in the
    350                      // heap_profile map of the heap when we get to it
    351   IGNORED_ON_HEAP,   // Is a live (ignored) object on heap
    352   MAYBE_LIVE,        // Is a piece of writable memory from /proc/self/maps
    353   IN_GLOBAL_DATA,    // Is part of global data region of the executable
    354   THREAD_DATA,       // Part of a thread stack and a thread descriptor with TLS
    355   THREAD_REGISTERS,  // Values in registers of some thread
    356 };
    357 
    358 // Information about an allocated object
    359 struct AllocObject {
    360   const void* ptr;        // the object
    361   uintptr_t size;         // its size
    362   ObjectPlacement place;  // where ptr points to
    363 
    364   AllocObject(const void* p, size_t s, ObjectPlacement l)
    365     : ptr(p), size(s), place(l) { }
    366 };
    367 
    368 // All objects (memory ranges) ignored via HeapLeakChecker::IgnoreObject
    369 // Key is the object's address; value is its size.
    370 typedef map<uintptr_t, size_t, less<uintptr_t>,
    371             STL_Allocator<pair<const uintptr_t, size_t>,
    372                           HeapLeakChecker::Allocator>
    373            > IgnoredObjectsMap;
    374 static IgnoredObjectsMap* ignored_objects = NULL;
    375 
    376 // All objects (memory ranges) that we consider to be the sources of pointers
    377 // to live (not leaked) objects.
    378 // At different times this holds (what can be reached from) global data regions
    379 // and the objects we've been told to ignore.
    380 // For any AllocObject::ptr "live_objects" is supposed to contain at most one
    381 // record at any time. We maintain this by checking with the heap_profile map
    382 // of the heap and removing the live heap objects we've handled from it.
    383 // This vector is maintained as a stack and the frontier of reachable
    384 // live heap objects in our flood traversal of them.
    385 typedef vector<AllocObject,
    386                STL_Allocator<AllocObject, HeapLeakChecker::Allocator>
    387               > LiveObjectsStack;
    388 static LiveObjectsStack* live_objects = NULL;
    389 
    390 // A special string type that uses my allocator
    391 typedef basic_string<char, char_traits<char>,
    392                      STL_Allocator<char, HeapLeakChecker::Allocator>
    393                     > HCL_string;
    394 
    395 // A placeholder to fill-in the starting values for live_objects
    396 // for each library so we can keep the library-name association for logging.
    397 typedef map<HCL_string, LiveObjectsStack, less<HCL_string>,
    398             STL_Allocator<pair<const HCL_string, LiveObjectsStack>,
    399                           HeapLeakChecker::Allocator>
    400            > LibraryLiveObjectsStacks;
    401 static LibraryLiveObjectsStacks* library_live_objects = NULL;
    402 
    403 // Value stored in the map of disabled address ranges;
    404 // its key is the end of the address range.
    405 // We'll ignore allocations with a return address in a disabled range
    406 // if the address occurs at 'max_depth' or less in the stack trace.
    407 struct HeapLeakChecker::RangeValue {
    408   uintptr_t start_address;  // the start of the range
    409   int       max_depth;      // the maximal stack depth to disable at
    410 };
    411 typedef map<uintptr_t, HeapLeakChecker::RangeValue, less<uintptr_t>,
    412             STL_Allocator<pair<const uintptr_t, HeapLeakChecker::RangeValue>,
    413                           HeapLeakChecker::Allocator>
    414            > DisabledRangeMap;
    415 // The disabled program counter address ranges for profile dumping
    416 // that are registered with HeapLeakChecker::DisableChecksFromToLocked.
    417 static DisabledRangeMap* disabled_ranges = NULL;
    418 
    419 // Set of stack tops.
    420 // These are used to consider live only appropriate chunks of the memory areas
    421 // that are used for stacks (and maybe thread-specific data as well)
    422 // so that we do not treat pointers from outdated stack frames as live.
    423 typedef set<uintptr_t, less<uintptr_t>,
    424             STL_Allocator<uintptr_t, HeapLeakChecker::Allocator>
    425            > StackTopSet;
    426 static StackTopSet* stack_tops = NULL;
    427 
    428 // A map of ranges of code addresses for the system libraries
    429 // that can mmap/mremap/sbrk-allocate memory regions for stacks
    430 // and thread-local storage that we want to consider as live global data.
    431 // Maps from the end address to the start address.
    432 typedef map<uintptr_t, uintptr_t, less<uintptr_t>,
    433             STL_Allocator<pair<const uintptr_t, uintptr_t>,
    434                           HeapLeakChecker::Allocator>
    435            > GlobalRegionCallerRangeMap;
    436 static GlobalRegionCallerRangeMap* global_region_caller_ranges = NULL;
    437 
    438 // TODO(maxim): make our big data structs into own modules
    439 
    440 // Disabler is implemented by keeping track of a per-thread count
    441 // of active Disabler objects.  Any objects allocated while the
    442 // count > 0 are not reported.
    443 
    444 #ifdef HAVE_TLS
    445 
    446 static __thread int thread_disable_counter
    447 // The "inital exec" model is faster than the default TLS model, at
    448 // the cost you can't dlopen this library.  But dlopen on heap-checker
    449 // doesn't work anyway -- it must run before main -- so this is a good
    450 // trade-off.
    451 # ifdef HAVE___ATTRIBUTE__
    452    __attribute__ ((tls_model ("initial-exec")))
    453 # endif
    454     ;
    455 inline int get_thread_disable_counter() {
    456   return thread_disable_counter;
    457 }
    458 inline void set_thread_disable_counter(int value) {
    459   thread_disable_counter = value;
    460 }
    461 
    462 #else  // #ifdef HAVE_TLS
    463 
    464 static pthread_key_t thread_disable_counter_key;
    465 static int main_thread_counter;   // storage for use before main()
    466 static bool use_main_thread_counter = true;
    467 
    468 // TODO(csilvers): this is called from NewHook, in the middle of malloc().
    469 // If perftools_pthread_getspecific calls malloc, that will lead to an
    470 // infinite loop.  I don't know how to fix that, so I hope it never happens!
    471 inline int get_thread_disable_counter() {
    472   if (use_main_thread_counter)  // means we're running really early
    473     return main_thread_counter;
    474   void* p = perftools_pthread_getspecific(thread_disable_counter_key);
    475   return (intptr_t)p;   // kinda evil: store the counter directly in the void*
    476 }
    477 
    478 inline void set_thread_disable_counter(int value) {
    479   if (use_main_thread_counter) {   // means we're running really early
    480     main_thread_counter = value;
    481     return;
    482   }
    483   intptr_t pointer_sized_value = value;
    484   // kinda evil: store the counter directly in the void*
    485   void* p = (void*)pointer_sized_value;
    486   // NOTE: this may call malloc, which will call NewHook which will call
    487   // get_thread_disable_counter() which will call pthread_getspecific().  I
    488   // don't know if anything bad can happen if we call getspecific() in the
    489   // middle of a setspecific() call.  It seems to work ok in practice...
    490   perftools_pthread_setspecific(thread_disable_counter_key, p);
    491 }
    492 
    493 // The idea here is that this initializer will run pretty late: after
    494 // pthreads have been totally set up.  At this point we can call
    495 // pthreads routines, so we set those up.
    496 class InitThreadDisableCounter {
    497  public:
    498   InitThreadDisableCounter() {
    499     perftools_pthread_key_create(&thread_disable_counter_key, NULL);
    500     // Set up the main thread's value, which we have a special variable for.
    501     void* p = (void*)main_thread_counter;   // store the counter directly
    502     perftools_pthread_setspecific(thread_disable_counter_key, p);
    503     use_main_thread_counter = false;
    504   }
    505 };
    506 InitThreadDisableCounter init_thread_disable_counter;
    507 
    508 #endif  // #ifdef HAVE_TLS
    509 
    510 HeapLeakChecker::Disabler::Disabler() {
    511   // It is faster to unconditionally increment the thread-local
    512   // counter than to check whether or not heap-checking is on
    513   // in a thread-safe manner.
    514   int counter = get_thread_disable_counter();
    515   set_thread_disable_counter(counter + 1);
    516   RAW_VLOG(10, "Increasing thread disable counter to %d", counter + 1);
    517 }
    518 
    519 HeapLeakChecker::Disabler::~Disabler() {
    520   int counter = get_thread_disable_counter();
    521   RAW_DCHECK(counter > 0, "");
    522   if (counter > 0) {
    523     set_thread_disable_counter(counter - 1);
    524     RAW_VLOG(10, "Decreasing thread disable counter to %d", counter);
    525   } else {
    526     RAW_VLOG(0, "Thread disable counter underflow : %d", counter);
    527   }
    528 }
    529 
    530 //----------------------------------------------------------------------
    531 
    532 // The size of the largest heap object allocated so far.
    533 static size_t max_heap_object_size = 0;
    534 // The possible range of addresses that can point
    535 // into one of the elements of heap_objects.
    536 static uintptr_t min_heap_address = uintptr_t(-1LL);
    537 static uintptr_t max_heap_address = 0;
    538 
    539 //----------------------------------------------------------------------
    540 
    541 // Simple casting helpers for uintptr_t and void*:
    542 template<typename T>
    543 inline static const void* AsPtr(T addr) {
    544   return reinterpret_cast<void*>(addr);
    545 }
    546 inline static uintptr_t AsInt(const void* ptr) {
    547   return reinterpret_cast<uintptr_t>(ptr);
    548 }
    549 
    550 //----------------------------------------------------------------------
    551 
    552 // We've seen reports that strstr causes heap-checker crashes in some
    553 // libc's (?):
    554 //    http://code.google.com/p/gperftools/issues/detail?id=263
    555 // It's simple enough to use our own.  This is not in time-critical code.
    556 static const char* hc_strstr(const char* s1, const char* s2) {
    557   const size_t len = strlen(s2);
    558   RAW_CHECK(len > 0, "Unexpected empty string passed to strstr()");
    559   for (const char* p = strchr(s1, *s2); p != NULL; p = strchr(p+1, *s2)) {
    560     if (strncmp(p, s2, len) == 0) {
    561       return p;
    562     }
    563   }
    564   return NULL;
    565 }
    566 
    567 //----------------------------------------------------------------------
    568 
    569 // Our hooks for MallocHook
    570 static void NewHook(const void* ptr, size_t size) {
    571   if (ptr != NULL) {
    572     const int counter = get_thread_disable_counter();
    573     const bool ignore = (counter > 0);
    574     RAW_VLOG(16, "Recording Alloc: %p of %"PRIuS "; %d", ptr, size,
    575              int(counter));
    576 
    577     // Fetch the caller's stack trace before acquiring heap_checker_lock.
    578     void* stack[HeapProfileTable::kMaxStackDepth];
    579     int depth = HeapProfileTable::GetCallerStackTrace(0, stack);
    580 
    581     { SpinLockHolder l(&heap_checker_lock);
    582       if (size > max_heap_object_size) max_heap_object_size = size;
    583       uintptr_t addr = AsInt(ptr);
    584       if (addr < min_heap_address) min_heap_address = addr;
    585       addr += size;
    586       if (addr > max_heap_address) max_heap_address = addr;
    587       if (heap_checker_on) {
    588         heap_profile->RecordAlloc(ptr, size, depth, stack);
    589         if (ignore) {
    590           heap_profile->MarkAsIgnored(ptr);
    591         }
    592       }
    593     }
    594     RAW_VLOG(17, "Alloc Recorded: %p of %"PRIuS"", ptr, size);
    595   }
    596 }
    597 
    598 static void DeleteHook(const void* ptr) {
    599   if (ptr != NULL) {
    600     RAW_VLOG(16, "Recording Free %p", ptr);
    601     { SpinLockHolder l(&heap_checker_lock);
    602       if (heap_checker_on) heap_profile->RecordFree(ptr);
    603     }
    604     RAW_VLOG(17, "Free Recorded: %p", ptr);
    605   }
    606 }
    607 
    608 //----------------------------------------------------------------------
    609 
    610 enum StackDirection {
    611   GROWS_TOWARDS_HIGH_ADDRESSES,
    612   GROWS_TOWARDS_LOW_ADDRESSES,
    613   UNKNOWN_DIRECTION
    614 };
    615 
    616 // Determine which way the stack grows:
    617 
    618 static StackDirection ATTRIBUTE_NOINLINE GetStackDirection(
    619     const uintptr_t *const ptr) {
    620   uintptr_t x;
    621   if (&x < ptr)
    622     return GROWS_TOWARDS_LOW_ADDRESSES;
    623   if (ptr < &x)
    624     return GROWS_TOWARDS_HIGH_ADDRESSES;
    625 
    626   RAW_CHECK(0, "");  // Couldn't determine the stack direction.
    627 
    628   return UNKNOWN_DIRECTION;
    629 }
    630 
    631 // Direction of stack growth (will initialize via GetStackDirection())
    632 static StackDirection stack_direction = UNKNOWN_DIRECTION;
    633 
    634 // This routine is called for every thread stack we know about to register it.
    635 static void RegisterStackLocked(const void* top_ptr) {
    636   RAW_DCHECK(heap_checker_lock.IsHeld(), "");
    637   RAW_DCHECK(MemoryRegionMap::LockIsHeld(), "");
    638   RAW_VLOG(10, "Thread stack at %p", top_ptr);
    639   uintptr_t top = AsInt(top_ptr);
    640   stack_tops->insert(top);  // add for later use
    641 
    642   // make sure stack_direction is initialized
    643   if (stack_direction == UNKNOWN_DIRECTION) {
    644     stack_direction = GetStackDirection(&top);
    645   }
    646 
    647   // Find memory region with this stack
    648   MemoryRegionMap::Region region;
    649   if (MemoryRegionMap::FindAndMarkStackRegion(top, &region)) {
    650     // Make the proper portion of the stack live:
    651     if (stack_direction == GROWS_TOWARDS_LOW_ADDRESSES) {
    652       RAW_VLOG(11, "Live stack at %p of %"PRIuPTR" bytes",
    653                   top_ptr, region.end_addr - top);
    654       live_objects->push_back(AllocObject(top_ptr, region.end_addr - top,
    655                                           THREAD_DATA));
    656     } else {  // GROWS_TOWARDS_HIGH_ADDRESSES
    657       RAW_VLOG(11, "Live stack at %p of %"PRIuPTR" bytes",
    658                   AsPtr(region.start_addr),
    659                   top - region.start_addr);
    660       live_objects->push_back(AllocObject(AsPtr(region.start_addr),
    661                                           top - region.start_addr,
    662                                           THREAD_DATA));
    663     }
    664   // not in MemoryRegionMap, look in library_live_objects:
    665   } else if (FLAGS_heap_check_ignore_global_live) {
    666     for (LibraryLiveObjectsStacks::iterator lib = library_live_objects->begin();
    667          lib != library_live_objects->end(); ++lib) {
    668       for (LiveObjectsStack::iterator span = lib->second.begin();
    669            span != lib->second.end(); ++span) {
    670         uintptr_t start = AsInt(span->ptr);
    671         uintptr_t end = start + span->size;
    672         if (start <= top  &&  top < end) {
    673           RAW_VLOG(11, "Stack at %p is inside /proc/self/maps chunk %p..%p",
    674                       top_ptr, AsPtr(start), AsPtr(end));
    675           // Shrink start..end region by chopping away the memory regions in
    676           // MemoryRegionMap that land in it to undo merging of regions
    677           // in /proc/self/maps, so that we correctly identify what portion
    678           // of start..end is actually the stack region.
    679           uintptr_t stack_start = start;
    680           uintptr_t stack_end = end;
    681           // can optimize-away this loop, but it does not run often
    682           RAW_DCHECK(MemoryRegionMap::LockIsHeld(), "");
    683           for (MemoryRegionMap::RegionIterator r =
    684                  MemoryRegionMap::BeginRegionLocked();
    685                r != MemoryRegionMap::EndRegionLocked(); ++r) {
    686             if (top < r->start_addr  &&  r->start_addr < stack_end) {
    687               stack_end = r->start_addr;
    688             }
    689             if (stack_start < r->end_addr  &&  r->end_addr <= top) {
    690               stack_start = r->end_addr;
    691             }
    692           }
    693           if (stack_start != start  ||  stack_end != end) {
    694             RAW_VLOG(11, "Stack at %p is actually inside memory chunk %p..%p",
    695                         top_ptr, AsPtr(stack_start), AsPtr(stack_end));
    696           }
    697           // Make the proper portion of the stack live:
    698           if (stack_direction == GROWS_TOWARDS_LOW_ADDRESSES) {
    699             RAW_VLOG(11, "Live stack at %p of %"PRIuPTR" bytes",
    700                         top_ptr, stack_end - top);
    701             live_objects->push_back(
    702               AllocObject(top_ptr, stack_end - top, THREAD_DATA));
    703           } else {  // GROWS_TOWARDS_HIGH_ADDRESSES
    704             RAW_VLOG(11, "Live stack at %p of %"PRIuPTR" bytes",
    705                         AsPtr(stack_start), top - stack_start);
    706             live_objects->push_back(
    707               AllocObject(AsPtr(stack_start), top - stack_start, THREAD_DATA));
    708           }
    709           lib->second.erase(span);  // kill the rest of the region
    710           // Put the non-stack part(s) of the region back:
    711           if (stack_start != start) {
    712             lib->second.push_back(AllocObject(AsPtr(start), stack_start - start,
    713                                   MAYBE_LIVE));
    714           }
    715           if (stack_end != end) {
    716             lib->second.push_back(AllocObject(AsPtr(stack_end), end - stack_end,
    717                                   MAYBE_LIVE));
    718           }
    719           return;
    720         }
    721       }
    722     }
    723     RAW_LOG(ERROR, "Memory region for stack at %p not found. "
    724                    "Will likely report false leak positives.", top_ptr);
    725   }
    726 }
    727 
    728 // Iterator for heap allocation map data to make ignored objects "live"
    729 // (i.e., treated as roots for the mark-and-sweep phase)
    730 static void MakeIgnoredObjectsLiveCallbackLocked(
    731     const void* ptr, const HeapProfileTable::AllocInfo& info) {
    732   RAW_DCHECK(heap_checker_lock.IsHeld(), "");
    733   if (info.ignored) {
    734     live_objects->push_back(AllocObject(ptr, info.object_size,
    735                                         MUST_BE_ON_HEAP));
    736   }
    737 }
    738 
    739 // Iterator for heap allocation map data to make objects allocated from
    740 // disabled regions of code to be live.
    741 static void MakeDisabledLiveCallbackLocked(
    742     const void* ptr, const HeapProfileTable::AllocInfo& info) {
    743   RAW_DCHECK(heap_checker_lock.IsHeld(), "");
    744   bool stack_disable = false;
    745   bool range_disable = false;
    746   for (int depth = 0; depth < info.stack_depth; depth++) {
    747     uintptr_t addr = AsInt(info.call_stack[depth]);
    748     if (disabled_ranges) {
    749       DisabledRangeMap::const_iterator iter
    750         = disabled_ranges->upper_bound(addr);
    751       if (iter != disabled_ranges->end()) {
    752         RAW_DCHECK(iter->first > addr, "");
    753         if (iter->second.start_address < addr  &&
    754             iter->second.max_depth > depth) {
    755           range_disable = true;  // in range; dropping
    756           break;
    757         }
    758       }
    759     }
    760   }
    761   if (stack_disable || range_disable) {
    762     uintptr_t start_address = AsInt(ptr);
    763     uintptr_t end_address = start_address + info.object_size;
    764     StackTopSet::const_iterator iter
    765       = stack_tops->lower_bound(start_address);
    766     if (iter != stack_tops->end()) {
    767       RAW_DCHECK(*iter >= start_address, "");
    768       if (*iter < end_address) {
    769         // We do not disable (treat as live) whole allocated regions
    770         // if they are used to hold thread call stacks
    771         // (i.e. when we find a stack inside).
    772         // The reason is that we'll treat as live the currently used
    773         // stack portions anyway (see RegisterStackLocked),
    774         // and the rest of the region where the stack lives can well
    775         // contain outdated stack variables which are not live anymore,
    776         // hence should not be treated as such.
    777         RAW_VLOG(11, "Not %s-disabling %"PRIuS" bytes at %p"
    778                     ": have stack inside: %p",
    779                     (stack_disable ? "stack" : "range"),
    780                     info.object_size, ptr, AsPtr(*iter));
    781         return;
    782       }
    783     }
    784     RAW_VLOG(11, "%s-disabling %"PRIuS" bytes at %p",
    785                 (stack_disable ? "Stack" : "Range"), info.object_size, ptr);
    786     live_objects->push_back(AllocObject(ptr, info.object_size,
    787                                         MUST_BE_ON_HEAP));
    788   }
    789 }
    790 
    791 static const char kUnnamedProcSelfMapEntry[] = "UNNAMED";
    792 
    793 // This function takes some fields from a /proc/self/maps line:
    794 //
    795 //   start_address  start address of a memory region.
    796 //   end_address    end address of a memory region
    797 //   permissions    rwx + private/shared bit
    798 //   filename       filename of the mapped file
    799 //
    800 // If the region is not writeable, then it cannot have any heap
    801 // pointers in it, otherwise we record it as a candidate live region
    802 // to get filtered later.
    803 static void RecordGlobalDataLocked(uintptr_t start_address,
    804                                    uintptr_t end_address,
    805                                    const char* permissions,
    806                                    const char* filename) {
    807   RAW_DCHECK(heap_checker_lock.IsHeld(), "");
    808   // Ignore non-writeable regions.
    809   if (strchr(permissions, 'w') == NULL) return;
    810   if (filename == NULL  ||  *filename == '\0') {
    811     filename = kUnnamedProcSelfMapEntry;
    812   }
    813   RAW_VLOG(11, "Looking into %s: 0x%" PRIxPTR "..0x%" PRIxPTR,
    814               filename, start_address, end_address);
    815   (*library_live_objects)[filename].
    816     push_back(AllocObject(AsPtr(start_address),
    817                           end_address - start_address,
    818                           MAYBE_LIVE));
    819 }
    820 
    821 // See if 'library' from /proc/self/maps has base name 'library_base'
    822 // i.e. contains it and has '.' or '-' after it.
    823 static bool IsLibraryNamed(const char* library, const char* library_base) {
    824   const char* p = hc_strstr(library, library_base);
    825   size_t sz = strlen(library_base);
    826   return p != NULL  &&  (p[sz] == '.'  ||  p[sz] == '-');
    827 }
    828 
    829 // static
    830 void HeapLeakChecker::DisableLibraryAllocsLocked(const char* library,
    831                                                  uintptr_t start_address,
    832                                                  uintptr_t end_address) {
    833   RAW_DCHECK(heap_checker_lock.IsHeld(), "");
    834   int depth = 0;
    835   // TODO(maxim): maybe this should be extended to also use objdump
    836   //              and pick the text portion of the library more precisely.
    837   if (IsLibraryNamed(library, "/libpthread")  ||
    838         // libpthread has a lot of small "system" leaks we don't care about.
    839         // In particular it allocates memory to store data supplied via
    840         // pthread_setspecific (which can be the only pointer to a heap object).
    841       IsLibraryNamed(library, "/libdl")  ||
    842         // library loaders leak some "system" heap that we don't care about
    843       IsLibraryNamed(library, "/libcrypto")  ||
    844         // Sometimes libcrypto of OpenSSH is compiled with -fomit-frame-pointer
    845         // (any library can be, of course, but this one often is because speed
    846         // is so important for making crypto usable).  We ignore all its
    847         // allocations because we can't see the call stacks.  We'd prefer
    848         // to ignore allocations done in files/symbols that match
    849         // "default_malloc_ex|default_realloc_ex"
    850         // but that doesn't work when the end-result binary is stripped.
    851       IsLibraryNamed(library, "/libjvm")  ||
    852         // JVM has a lot of leaks we don't care about.
    853       IsLibraryNamed(library, "/libzip")
    854         // The JVM leaks java.util.zip.Inflater after loading classes.
    855      ) {
    856     depth = 1;  // only disable allocation calls directly from the library code
    857   } else if (IsLibraryNamed(library, "/ld")
    858                // library loader leaks some "system" heap
    859                // (e.g. thread-local storage) that we don't care about
    860             ) {
    861     depth = 2;  // disable allocation calls directly from the library code
    862                 // and at depth 2 from it.
    863     // We need depth 2 here solely because of a libc bug that
    864     // forces us to jump through __memalign_hook and MemalignOverride hoops
    865     // in tcmalloc.cc.
    866     // Those buggy __libc_memalign() calls are in ld-linux.so and happen for
    867     // thread-local storage allocations that we want to ignore here.
    868     // We go with the depth-2 hack as a workaround for this libc bug:
    869     // otherwise we'd need to extend MallocHook interface
    870     // so that correct stack depth adjustment can be propagated from
    871     // the exceptional case of MemalignOverride.
    872     // Using depth 2 here should not mask real leaks because ld-linux.so
    873     // does not call user code.
    874   }
    875   if (depth) {
    876     RAW_VLOG(10, "Disabling allocations from %s at depth %d:", library, depth);
    877     DisableChecksFromToLocked(AsPtr(start_address), AsPtr(end_address), depth);
    878     if (IsLibraryNamed(library, "/libpthread")  ||
    879         IsLibraryNamed(library, "/libdl")  ||
    880         IsLibraryNamed(library, "/ld")) {
    881       RAW_VLOG(10, "Global memory regions made by %s will be live data",
    882                   library);
    883       if (global_region_caller_ranges == NULL) {
    884         global_region_caller_ranges =
    885           new(Allocator::Allocate(sizeof(GlobalRegionCallerRangeMap)))
    886             GlobalRegionCallerRangeMap;
    887       }
    888       global_region_caller_ranges
    889         ->insert(make_pair(end_address, start_address));
    890     }
    891   }
    892 }
    893 
    894 // static
    895 HeapLeakChecker::ProcMapsResult HeapLeakChecker::UseProcMapsLocked(
    896                                   ProcMapsTask proc_maps_task) {
    897   RAW_DCHECK(heap_checker_lock.IsHeld(), "");
    898   // Need to provide own scratch memory to ProcMapsIterator:
    899   ProcMapsIterator::Buffer buffer;
    900   ProcMapsIterator it(0, &buffer);
    901   if (!it.Valid()) {
    902     int errsv = errno;
    903     RAW_LOG(ERROR, "Could not open /proc/self/maps: errno=%d. "
    904                    "Libraries will not be handled correctly.", errsv);
    905     return CANT_OPEN_PROC_MAPS;
    906   }
    907   uint64 start_address, end_address, file_offset;
    908   int64 inode;
    909   char *permissions, *filename;
    910   bool saw_shared_lib = false;
    911   bool saw_nonzero_inode = false;
    912   bool saw_shared_lib_with_nonzero_inode = false;
    913   while (it.Next(&start_address, &end_address, &permissions,
    914                  &file_offset, &inode, &filename)) {
    915     if (start_address >= end_address) {
    916       // Warn if a line we can be interested in is ill-formed:
    917       if (inode != 0) {
    918         RAW_LOG(ERROR, "Errors reading /proc/self/maps. "
    919                        "Some global memory regions will not "
    920                        "be handled correctly.");
    921       }
    922       // Silently skip other ill-formed lines: some are possible
    923       // probably due to the interplay of how /proc/self/maps is updated
    924       // while we read it in chunks in ProcMapsIterator and
    925       // do things in this loop.
    926       continue;
    927     }
    928     // Determine if any shared libraries are present (this is the same
    929     // list of extensions as is found in pprof).  We want to ignore
    930     // 'fake' libraries with inode 0 when determining.  However, some
    931     // systems don't share inodes via /proc, so we turn off this check
    932     // if we don't see any evidence that we're getting inode info.
    933     if (inode != 0) {
    934       saw_nonzero_inode = true;
    935     }
    936     if ((hc_strstr(filename, "lib") && hc_strstr(filename, ".so")) ||
    937         hc_strstr(filename, ".dll") ||
    938         // not all .dylib filenames start with lib. .dylib is big enough
    939         // that we are unlikely to get false matches just checking that.
    940         hc_strstr(filename, ".dylib") || hc_strstr(filename, ".bundle")) {
    941       saw_shared_lib = true;
    942       if (inode != 0) {
    943         saw_shared_lib_with_nonzero_inode = true;
    944       }
    945     }
    946 
    947     switch (proc_maps_task) {
    948       case DISABLE_LIBRARY_ALLOCS:
    949         // All lines starting like
    950         // "401dc000-4030f000 r??p 00132000 03:01 13991972  lib/bin"
    951         // identify a data and code sections of a shared library or our binary
    952         if (inode != 0 && strncmp(permissions, "r-xp", 4) == 0) {
    953           DisableLibraryAllocsLocked(filename, start_address, end_address);
    954         }
    955         break;
    956       case RECORD_GLOBAL_DATA:
    957         RecordGlobalDataLocked(start_address, end_address,
    958                                permissions, filename);
    959         break;
    960       default:
    961         RAW_CHECK(0, "");
    962     }
    963   }
    964   // If /proc/self/maps is reporting inodes properly (we saw a
    965   // non-zero inode), then we only say we saw a shared lib if we saw a
    966   // 'real' one, with a non-zero inode.
    967   if (saw_nonzero_inode) {
    968     saw_shared_lib = saw_shared_lib_with_nonzero_inode;
    969   }
    970   if (!saw_shared_lib) {
    971     RAW_LOG(ERROR, "No shared libs detected. Will likely report false leak "
    972                    "positives for statically linked executables.");
    973     return NO_SHARED_LIBS_IN_PROC_MAPS;
    974   }
    975   return PROC_MAPS_USED;
    976 }
    977 
    978 // Total number and size of live objects dropped from the profile;
    979 // (re)initialized in IgnoreAllLiveObjectsLocked.
    980 static int64 live_objects_total;
    981 static int64 live_bytes_total;
    982 
    983 // pid of the thread that is doing the current leak check
    984 // (protected by our lock; IgnoreAllLiveObjectsLocked sets it)
    985 static pid_t self_thread_pid = 0;
    986 
    987 // Status of our thread listing callback execution
    988 // (protected by our lock; used from within IgnoreAllLiveObjectsLocked)
    989 static enum {
    990   CALLBACK_NOT_STARTED,
    991   CALLBACK_STARTED,
    992   CALLBACK_COMPLETED,
    993 } thread_listing_status = CALLBACK_NOT_STARTED;
    994 
    995 // Ideally to avoid deadlocks this function should not result in any libc
    996 // or other function calls that might need to lock a mutex:
    997 // It is called when all threads of a process are stopped
    998 // at arbitrary points thus potentially holding those locks.
    999 //
   1000 // In practice we are calling some simple i/o and sprintf-type library functions
   1001 // for logging messages, but use only our own LowLevelAlloc::Arena allocator.
   1002 //
   1003 // This is known to be buggy: the library i/o function calls are able to cause
   1004 // deadlocks when they request a lock that a stopped thread happens to hold.
   1005 // This issue as far as we know have so far not resulted in any deadlocks
   1006 // in practice, so for now we are taking our chance that the deadlocks
   1007 // have insignificant frequency.
   1008 //
   1009 // If such deadlocks become a problem we should make the i/o calls
   1010 // into appropriately direct system calls (or eliminate them),
   1011 // in particular write() is not safe and vsnprintf() is potentially dangerous
   1012 // due to reliance on locale functions (these are called through RAW_LOG
   1013 // and in other ways).
   1014 //
   1015 /*static*/ int HeapLeakChecker::IgnoreLiveThreadsLocked(void* parameter,
   1016                                                         int num_threads,
   1017                                                         pid_t* thread_pids,
   1018                                                         va_list /*ap*/) {
   1019   RAW_DCHECK(heap_checker_lock.IsHeld(), "");
   1020   thread_listing_status = CALLBACK_STARTED;
   1021   RAW_VLOG(11, "Found %d threads (from pid %d)", num_threads, getpid());
   1022 
   1023   if (FLAGS_heap_check_ignore_global_live) {
   1024     UseProcMapsLocked(RECORD_GLOBAL_DATA);
   1025   }
   1026 
   1027   // We put the registers from other threads here
   1028   // to make pointers stored in them live.
   1029   vector<void*, STL_Allocator<void*, Allocator> > thread_registers;
   1030 
   1031   int failures = 0;
   1032   for (int i = 0; i < num_threads; ++i) {
   1033     // the leak checking thread itself is handled
   1034     // specially via self_thread_stack, not here:
   1035     if (thread_pids[i] == self_thread_pid) continue;
   1036     RAW_VLOG(11, "Handling thread with pid %d", thread_pids[i]);
   1037 #if (defined(__i386__) || defined(__x86_64)) && \
   1038     defined(HAVE_LINUX_PTRACE_H) && defined(HAVE_SYS_SYSCALL_H) && defined(DUMPER)
   1039     i386_regs thread_regs;
   1040 #define sys_ptrace(r, p, a, d)  syscall(SYS_ptrace, (r), (p), (a), (d))
   1041     // We use sys_ptrace to avoid thread locking
   1042     // because this is called from ListAllProcessThreads
   1043     // when all but this thread are suspended.
   1044     if (sys_ptrace(PTRACE_GETREGS, thread_pids[i], NULL, &thread_regs) == 0) {
   1045       // Need to use SP to get all the data from the very last stack frame:
   1046       COMPILE_ASSERT(sizeof(thread_regs.SP) == sizeof(void*),
   1047                      SP_register_does_not_look_like_a_pointer);
   1048       RegisterStackLocked(reinterpret_cast<void*>(thread_regs.SP));
   1049       // Make registers live (just in case PTRACE_ATTACH resulted in some
   1050       // register pointers still being in the registers and not on the stack):
   1051       for (void** p = reinterpret_cast<void**>(&thread_regs);
   1052            p < reinterpret_cast<void**>(&thread_regs + 1); ++p) {
   1053         RAW_VLOG(12, "Thread register %p", *p);
   1054         thread_registers.push_back(*p);
   1055       }
   1056     } else {
   1057       failures += 1;
   1058     }
   1059 #else
   1060     failures += 1;
   1061 #endif
   1062   }
   1063   // Use all the collected thread (stack) liveness sources:
   1064   IgnoreLiveObjectsLocked("threads stack data", "");
   1065   if (thread_registers.size()) {
   1066     // Make thread registers be live heap data sources.
   1067     // we rely here on the fact that vector is in one memory chunk:
   1068     RAW_VLOG(11, "Live registers at %p of %"PRIuS" bytes",
   1069                 &thread_registers[0], thread_registers.size() * sizeof(void*));
   1070     live_objects->push_back(AllocObject(&thread_registers[0],
   1071                                         thread_registers.size() * sizeof(void*),
   1072                                         THREAD_REGISTERS));
   1073     IgnoreLiveObjectsLocked("threads register data", "");
   1074   }
   1075   // Do all other liveness walking while all threads are stopped:
   1076   IgnoreNonThreadLiveObjectsLocked();
   1077   // Can now resume the threads:
   1078   ResumeAllProcessThreads(num_threads, thread_pids);
   1079   thread_listing_status = CALLBACK_COMPLETED;
   1080   return failures;
   1081 }
   1082 
   1083 // Stack top of the thread that is doing the current leak check
   1084 // (protected by our lock; IgnoreAllLiveObjectsLocked sets it)
   1085 static const void* self_thread_stack_top;
   1086 
   1087 // static
   1088 void HeapLeakChecker::IgnoreNonThreadLiveObjectsLocked() {
   1089   RAW_DCHECK(heap_checker_lock.IsHeld(), "");
   1090   RAW_DCHECK(MemoryRegionMap::LockIsHeld(), "");
   1091   RAW_VLOG(11, "Handling self thread with pid %d", self_thread_pid);
   1092   // Register our own stack:
   1093 
   1094   // Important that all stack ranges (including the one here)
   1095   // are known before we start looking at them
   1096   // in MakeDisabledLiveCallbackLocked:
   1097   RegisterStackLocked(self_thread_stack_top);
   1098   IgnoreLiveObjectsLocked("stack data", "");
   1099 
   1100   // Make objects we were told to ignore live:
   1101   if (ignored_objects) {
   1102     for (IgnoredObjectsMap::const_iterator object = ignored_objects->begin();
   1103          object != ignored_objects->end(); ++object) {
   1104       const void* ptr = AsPtr(object->first);
   1105       RAW_VLOG(11, "Ignored live object at %p of %"PRIuS" bytes",
   1106                   ptr, object->second);
   1107       live_objects->
   1108         push_back(AllocObject(ptr, object->second, MUST_BE_ON_HEAP));
   1109       // we do this liveness check for ignored_objects before doing any
   1110       // live heap walking to make sure it does not fail needlessly:
   1111       size_t object_size;
   1112       if (!(heap_profile->FindAlloc(ptr, &object_size)  &&
   1113             object->second == object_size)) {
   1114         RAW_LOG(FATAL, "Object at %p of %"PRIuS" bytes from an"
   1115                        " IgnoreObject() has disappeared", ptr, object->second);
   1116       }
   1117     }
   1118     IgnoreLiveObjectsLocked("ignored objects", "");
   1119   }
   1120 
   1121   // Treat objects that were allocated when a Disabler was live as
   1122   // roots.  I.e., if X was allocated while a Disabler was active,
   1123   // and Y is reachable from X, arrange that neither X nor Y are
   1124   // treated as leaks.
   1125   heap_profile->IterateAllocs(MakeIgnoredObjectsLiveCallbackLocked);
   1126   IgnoreLiveObjectsLocked("disabled objects", "");
   1127 
   1128   // Make code-address-disabled objects live and ignored:
   1129   // This in particular makes all thread-specific data live
   1130   // because the basic data structure to hold pointers to thread-specific data
   1131   // is allocated from libpthreads and we have range-disabled that
   1132   // library code with UseProcMapsLocked(DISABLE_LIBRARY_ALLOCS);
   1133   // so now we declare all thread-specific data reachable from there as live.
   1134   heap_profile->IterateAllocs(MakeDisabledLiveCallbackLocked);
   1135   IgnoreLiveObjectsLocked("disabled code", "");
   1136 
   1137   // Actually make global data live:
   1138   if (FLAGS_heap_check_ignore_global_live) {
   1139     bool have_null_region_callers = false;
   1140     for (LibraryLiveObjectsStacks::iterator l = library_live_objects->begin();
   1141          l != library_live_objects->end(); ++l) {
   1142       RAW_CHECK(live_objects->empty(), "");
   1143       // Process library_live_objects in l->second
   1144       // filtering them by MemoryRegionMap:
   1145       // It's safe to iterate over MemoryRegionMap
   1146       // w/o locks here as we are inside MemoryRegionMap::Lock():
   1147       RAW_DCHECK(MemoryRegionMap::LockIsHeld(), "");
   1148       // The only change to MemoryRegionMap possible in this loop
   1149       // is region addition as a result of allocating more memory
   1150       // for live_objects. This won't invalidate the RegionIterator
   1151       // or the intent of the loop.
   1152       // --see the comment by MemoryRegionMap::BeginRegionLocked().
   1153       for (MemoryRegionMap::RegionIterator region =
   1154              MemoryRegionMap::BeginRegionLocked();
   1155            region != MemoryRegionMap::EndRegionLocked(); ++region) {
   1156         // "region" from MemoryRegionMap is to be subtracted from
   1157         // (tentatively live) regions in l->second
   1158         // if it has a stack inside or it was allocated by
   1159         // a non-special caller (not one covered by a range
   1160         // in global_region_caller_ranges).
   1161         // This will in particular exclude all memory chunks used
   1162         // by the heap itself as well as what's been allocated with
   1163         // any allocator on top of mmap.
   1164         bool subtract = true;
   1165         if (!region->is_stack  &&  global_region_caller_ranges) {
   1166           if (region->caller() == static_cast<uintptr_t>(NULL)) {
   1167             have_null_region_callers = true;
   1168           } else {
   1169             GlobalRegionCallerRangeMap::const_iterator iter
   1170               = global_region_caller_ranges->upper_bound(region->caller());
   1171             if (iter != global_region_caller_ranges->end()) {
   1172               RAW_DCHECK(iter->first > region->caller(), "");
   1173               if (iter->second < region->caller()) {  // in special region
   1174                 subtract = false;
   1175               }
   1176             }
   1177           }
   1178         }
   1179         if (subtract) {
   1180           // The loop puts the result of filtering l->second into live_objects:
   1181           for (LiveObjectsStack::const_iterator i = l->second.begin();
   1182                i != l->second.end(); ++i) {
   1183             // subtract *region from *i
   1184             uintptr_t start = AsInt(i->ptr);
   1185             uintptr_t end = start + i->size;
   1186             if (region->start_addr <= start  &&  end <= region->end_addr) {
   1187               // full deletion due to subsumption
   1188             } else if (start < region->start_addr  &&
   1189                        region->end_addr < end) {  // cutting-out split
   1190               live_objects->push_back(AllocObject(i->ptr,
   1191                                                   region->start_addr - start,
   1192                                                   IN_GLOBAL_DATA));
   1193               live_objects->push_back(AllocObject(AsPtr(region->end_addr),
   1194                                                   end - region->end_addr,
   1195                                                   IN_GLOBAL_DATA));
   1196             } else if (region->end_addr > start  &&
   1197                        region->start_addr <= start) {  // cut from start
   1198               live_objects->push_back(AllocObject(AsPtr(region->end_addr),
   1199                                                   end - region->end_addr,
   1200                                                   IN_GLOBAL_DATA));
   1201             } else if (region->start_addr > start  &&
   1202                        region->start_addr < end) {  // cut from end
   1203               live_objects->push_back(AllocObject(i->ptr,
   1204                                                   region->start_addr - start,
   1205                                                   IN_GLOBAL_DATA));
   1206             } else {  // pass: no intersection
   1207               live_objects->push_back(AllocObject(i->ptr, i->size,
   1208                                                   IN_GLOBAL_DATA));
   1209             }
   1210           }
   1211           // Move live_objects back into l->second
   1212           // for filtering by the next region.
   1213           live_objects->swap(l->second);
   1214           live_objects->clear();
   1215         }
   1216       }
   1217       // Now get and use live_objects from the final version of l->second:
   1218       if (VLOG_IS_ON(11)) {
   1219         for (LiveObjectsStack::const_iterator i = l->second.begin();
   1220              i != l->second.end(); ++i) {
   1221           RAW_VLOG(11, "Library live region at %p of %"PRIuPTR" bytes",
   1222                       i->ptr, i->size);
   1223         }
   1224       }
   1225       live_objects->swap(l->second);
   1226       IgnoreLiveObjectsLocked("in globals of\n  ", l->first.c_str());
   1227     }
   1228     if (have_null_region_callers) {
   1229       RAW_LOG(ERROR, "Have memory regions w/o callers: "
   1230                      "might report false leaks");
   1231     }
   1232     Allocator::DeleteAndNull(&library_live_objects);
   1233   }
   1234 }
   1235 
   1236 // Callback for ListAllProcessThreads in IgnoreAllLiveObjectsLocked below
   1237 // to test/verify that we have just the one main thread, in which case
   1238 // we can do everything in that main thread,
   1239 // so that CPU profiler can collect all its samples.
   1240 // Returns the number of threads in the process.
   1241 static int IsOneThread(void* parameter, int num_threads,
   1242                        pid_t* thread_pids, va_list ap) {
   1243   if (num_threads != 1) {
   1244     RAW_LOG(WARNING, "Have threads: Won't CPU-profile the bulk of leak "
   1245                      "checking work happening in IgnoreLiveThreadsLocked!");
   1246   }
   1247   ResumeAllProcessThreads(num_threads, thread_pids);
   1248   return num_threads;
   1249 }
   1250 
   1251 // Dummy for IgnoreAllLiveObjectsLocked below.
   1252 // Making it global helps with compiler warnings.
   1253 static va_list dummy_ap;
   1254 
   1255 // static
   1256 void HeapLeakChecker::IgnoreAllLiveObjectsLocked(const void* self_stack_top) {
   1257   RAW_DCHECK(heap_checker_lock.IsHeld(), "");
   1258   RAW_CHECK(live_objects == NULL, "");
   1259   live_objects = new(Allocator::Allocate(sizeof(LiveObjectsStack)))
   1260                    LiveObjectsStack;
   1261   stack_tops = new(Allocator::Allocate(sizeof(StackTopSet))) StackTopSet;
   1262   // reset the counts
   1263   live_objects_total = 0;
   1264   live_bytes_total = 0;
   1265   // Reduce max_heap_object_size to FLAGS_heap_check_max_pointer_offset
   1266   // for the time of leak check.
   1267   // FLAGS_heap_check_max_pointer_offset caps max_heap_object_size
   1268   // to manage reasonably low chances of random bytes
   1269   // appearing to be pointing into large actually leaked heap objects.
   1270   const size_t old_max_heap_object_size = max_heap_object_size;
   1271   max_heap_object_size = (
   1272     FLAGS_heap_check_max_pointer_offset != -1
   1273     ? min(size_t(FLAGS_heap_check_max_pointer_offset), max_heap_object_size)
   1274     : max_heap_object_size);
   1275   // Record global data as live:
   1276   if (FLAGS_heap_check_ignore_global_live) {
   1277     library_live_objects =
   1278       new(Allocator::Allocate(sizeof(LibraryLiveObjectsStacks)))
   1279         LibraryLiveObjectsStacks;
   1280   }
   1281   // Ignore all thread stacks:
   1282   thread_listing_status = CALLBACK_NOT_STARTED;
   1283   bool need_to_ignore_non_thread_objects = true;
   1284   self_thread_pid = getpid();
   1285   self_thread_stack_top = self_stack_top;
   1286   if (FLAGS_heap_check_ignore_thread_live) {
   1287     // In case we are doing CPU profiling we'd like to do all the work
   1288     // in the main thread, not in the special thread created by
   1289     // ListAllProcessThreads, so that CPU profiler can collect all its samples.
   1290     // The machinery of ListAllProcessThreads conflicts with the CPU profiler
   1291     // by also relying on signals and ::sigaction.
   1292     // We can do this (run everything in the main thread) safely
   1293     // only if there's just the main thread itself in our process.
   1294     // This variable reflects these two conditions:
   1295     bool want_and_can_run_in_main_thread =
   1296       ProfilingIsEnabledForAllThreads()  &&
   1297       ListAllProcessThreads(NULL, IsOneThread) == 1;
   1298     // When the normal path of ListAllProcessThreads below is taken,
   1299     // we fully suspend the threads right here before any liveness checking
   1300     // and keep them suspended for the whole time of liveness checking
   1301     // inside of the IgnoreLiveThreadsLocked callback.
   1302     // (The threads can't (de)allocate due to lock on the delete hook but
   1303     //  if not suspended they could still mess with the pointer
   1304     //  graph while we walk it).
   1305     int r = want_and_can_run_in_main_thread
   1306             ? IgnoreLiveThreadsLocked(NULL, 1, &self_thread_pid, dummy_ap)
   1307             : ListAllProcessThreads(NULL, IgnoreLiveThreadsLocked);
   1308     need_to_ignore_non_thread_objects = r < 0;
   1309     if (r < 0) {
   1310       RAW_LOG(WARNING, "Thread finding failed with %d errno=%d", r, errno);
   1311       if (thread_listing_status == CALLBACK_COMPLETED) {
   1312         RAW_LOG(INFO, "Thread finding callback "
   1313                       "finished ok; hopefully everything is fine");
   1314         need_to_ignore_non_thread_objects = false;
   1315       } else if (thread_listing_status == CALLBACK_STARTED) {
   1316         RAW_LOG(FATAL, "Thread finding callback was "
   1317                        "interrupted or crashed; can't fix this");
   1318       } else {  // CALLBACK_NOT_STARTED
   1319         RAW_LOG(ERROR, "Could not find thread stacks. "
   1320                        "Will likely report false leak positives.");
   1321       }
   1322     } else if (r != 0) {
   1323       RAW_LOG(ERROR, "Thread stacks not found for %d threads. "
   1324                      "Will likely report false leak positives.", r);
   1325     } else {
   1326       RAW_VLOG(11, "Thread stacks appear to be found for all threads");
   1327     }
   1328   } else {
   1329     RAW_LOG(WARNING, "Not looking for thread stacks; "
   1330                      "objects reachable only from there "
   1331                      "will be reported as leaks");
   1332   }
   1333   // Do all other live data ignoring here if we did not do it
   1334   // within thread listing callback with all threads stopped.
   1335   if (need_to_ignore_non_thread_objects) {
   1336     if (FLAGS_heap_check_ignore_global_live) {
   1337       UseProcMapsLocked(RECORD_GLOBAL_DATA);
   1338     }
   1339     IgnoreNonThreadLiveObjectsLocked();
   1340   }
   1341   if (live_objects_total) {
   1342     RAW_VLOG(10, "Ignoring %"PRId64" reachable objects of %"PRId64" bytes",
   1343                 live_objects_total, live_bytes_total);
   1344   }
   1345   // Free these: we made them here and heap_profile never saw them
   1346   Allocator::DeleteAndNull(&live_objects);
   1347   Allocator::DeleteAndNull(&stack_tops);
   1348   max_heap_object_size = old_max_heap_object_size;  // reset this var
   1349 }
   1350 
   1351 // Alignment at which we should consider pointer positions
   1352 // in IgnoreLiveObjectsLocked. Will normally use the value of
   1353 // FLAGS_heap_check_pointer_source_alignment.
   1354 static size_t pointer_source_alignment = kPointerSourceAlignment;
   1355 // Global lock for HeapLeakChecker::DoNoLeaks
   1356 // to protect pointer_source_alignment.
   1357 static SpinLock alignment_checker_lock(SpinLock::LINKER_INITIALIZED);
   1358 
   1359 // This function changes the live bits in the heap_profile-table's state:
   1360 // we only record the live objects to be skipped.
   1361 //
   1362 // When checking if a byte sequence points to a heap object we use
   1363 // HeapProfileTable::FindInsideAlloc to handle both pointers to
   1364 // the start and inside of heap-allocated objects.
   1365 // The "inside" case needs to be checked to support
   1366 // at least the following relatively common cases:
   1367 // - C++ arrays allocated with new FooClass[size] for classes
   1368 //   with destructors have their size recorded in a sizeof(int) field
   1369 //   before the place normal pointers point to.
   1370 // - basic_string<>-s for e.g. the C++ library of gcc 3.4
   1371 //   have the meta-info in basic_string<...>::_Rep recorded
   1372 //   before the place normal pointers point to.
   1373 // - Multiple-inherited objects have their pointers when cast to
   1374 //   different base classes pointing inside of the actually
   1375 //   allocated object.
   1376 // - Sometimes reachability pointers point to member objects of heap objects,
   1377 //   and then those member objects point to the full heap object.
   1378 // - Third party UnicodeString: it stores a 32-bit refcount
   1379 //   (in both 32-bit and 64-bit binaries) as the first uint32
   1380 //   in the allocated memory and a normal pointer points at
   1381 //   the second uint32 behind the refcount.
   1382 // By finding these additional objects here
   1383 // we slightly increase the chance to mistake random memory bytes
   1384 // for a pointer and miss a leak in a particular run of a binary.
   1385 //
   1386 /*static*/ void HeapLeakChecker::IgnoreLiveObjectsLocked(const char* name,
   1387                                                          const char* name2) {
   1388   RAW_DCHECK(heap_checker_lock.IsHeld(), "");
   1389   int64 live_object_count = 0;
   1390   int64 live_byte_count = 0;
   1391   while (!live_objects->empty()) {
   1392     const char* object =
   1393       reinterpret_cast<const char*>(live_objects->back().ptr);
   1394     size_t size = live_objects->back().size;
   1395     const ObjectPlacement place = live_objects->back().place;
   1396     live_objects->pop_back();
   1397     if (place == MUST_BE_ON_HEAP  &&  heap_profile->MarkAsLive(object)) {
   1398       live_object_count += 1;
   1399       live_byte_count += size;
   1400     }
   1401     RAW_VLOG(13, "Looking for heap pointers in %p of %"PRIuS" bytes",
   1402                 object, size);
   1403     const char* const whole_object = object;
   1404     size_t const whole_size = size;
   1405     // Try interpretting any byte sequence in object,size as a heap pointer:
   1406     const size_t remainder = AsInt(object) % pointer_source_alignment;
   1407     if (remainder) {
   1408       object += pointer_source_alignment - remainder;
   1409       if (size >= pointer_source_alignment - remainder) {
   1410         size -= pointer_source_alignment - remainder;
   1411       } else {
   1412         size = 0;
   1413       }
   1414     }
   1415     if (size < sizeof(void*)) continue;
   1416 
   1417 #ifdef NO_FRAME_POINTER
   1418     // Frame pointer omission requires us to use libunwind, which uses direct
   1419     // mmap and munmap system calls, and that needs special handling.
   1420     if (name2 == kUnnamedProcSelfMapEntry) {
   1421       static const uintptr_t page_mask = ~(getpagesize() - 1);
   1422       const uintptr_t addr = reinterpret_cast<uintptr_t>(object);
   1423       if ((addr & page_mask) == 0 && (size & page_mask) == 0) {
   1424         // This is an object we slurped from /proc/self/maps.
   1425         // It may or may not be readable at this point.
   1426         //
   1427         // In case all the above conditions made a mistake, and the object is
   1428         // not related to libunwind, we also verify that it's not readable
   1429         // before ignoring it.
   1430         if (msync(const_cast<char*>(object), size, MS_ASYNC) != 0) {
   1431           // Skip unreadable object, so we don't crash trying to sweep it.
   1432           RAW_VLOG(0, "Ignoring inaccessible object [%p, %p) "
   1433                    "(msync error %d (%s))",
   1434                    object, object + size, errno, strerror(errno));
   1435           continue;
   1436         }
   1437       }
   1438     }
   1439 #endif
   1440 
   1441     const char* const max_object = object + size - sizeof(void*);
   1442     while (object <= max_object) {
   1443       // potentially unaligned load:
   1444       const uintptr_t addr = *reinterpret_cast<const uintptr_t*>(object);
   1445       // Do fast check before the more expensive HaveOnHeapLocked lookup:
   1446       // this code runs for all memory words that are potentially pointers:
   1447       const bool can_be_on_heap =
   1448         // Order tests by the likelyhood of the test failing in 64/32 bit modes.
   1449         // Yes, this matters: we either lose 5..6% speed in 32 bit mode
   1450         // (which is already slower) or by a factor of 1.5..1.91 in 64 bit mode.
   1451         // After the alignment test got dropped the above performance figures
   1452         // must have changed; might need to revisit this.
   1453 #if defined(__x86_64__)
   1454         addr <= max_heap_address  &&  // <= is for 0-sized object with max addr
   1455         min_heap_address <= addr;
   1456 #else
   1457         min_heap_address <= addr  &&
   1458         addr <= max_heap_address;  // <= is for 0-sized object with max addr
   1459 #endif
   1460       if (can_be_on_heap) {
   1461         const void* ptr = reinterpret_cast<const void*>(addr);
   1462         // Too expensive (inner loop): manually uncomment when debugging:
   1463         // RAW_VLOG(17, "Trying pointer to %p at %p", ptr, object);
   1464         size_t object_size;
   1465         if (HaveOnHeapLocked(&ptr, &object_size)  &&
   1466             heap_profile->MarkAsLive(ptr)) {
   1467           // We take the (hopefully low) risk here of encountering by accident
   1468           // a byte sequence in memory that matches an address of
   1469           // a heap object which is in fact leaked.
   1470           // I.e. in very rare and probably not repeatable/lasting cases
   1471           // we might miss some real heap memory leaks.
   1472           RAW_VLOG(14, "Found pointer to %p of %"PRIuS" bytes at %p "
   1473                       "inside %p of size %"PRIuS"",
   1474                       ptr, object_size, object, whole_object, whole_size);
   1475           if (VLOG_IS_ON(15)) {
   1476             // log call stacks to help debug how come something is not a leak
   1477             HeapProfileTable::AllocInfo alloc;
   1478             bool r = heap_profile->FindAllocDetails(ptr, &alloc);
   1479             r = r;              // suppress compiler warning in non-debug mode
   1480             RAW_DCHECK(r, "");  // sanity
   1481             RAW_LOG(INFO, "New live %p object's alloc stack:", ptr);
   1482             for (int i = 0; i < alloc.stack_depth; ++i) {
   1483               RAW_LOG(INFO, "  @ %p", alloc.call_stack[i]);
   1484             }
   1485           }
   1486           live_object_count += 1;
   1487           live_byte_count += object_size;
   1488           live_objects->push_back(AllocObject(ptr, object_size,
   1489                                               IGNORED_ON_HEAP));
   1490         }
   1491       }
   1492       object += pointer_source_alignment;
   1493     }
   1494   }
   1495   live_objects_total += live_object_count;
   1496   live_bytes_total += live_byte_count;
   1497   if (live_object_count) {
   1498     RAW_VLOG(10, "Removed %"PRId64" live heap objects of %"PRId64" bytes: %s%s",
   1499                 live_object_count, live_byte_count, name, name2);
   1500   }
   1501 }
   1502 
   1503 //----------------------------------------------------------------------
   1504 // HeapLeakChecker leak check disabling components
   1505 //----------------------------------------------------------------------
   1506 
   1507 // static
   1508 void HeapLeakChecker::DisableChecksIn(const char* pattern) {
   1509   RAW_LOG(WARNING, "DisableChecksIn(%s) is ignored", pattern);
   1510 }
   1511 
   1512 // static
   1513 void HeapLeakChecker::DoIgnoreObject(const void* ptr) {
   1514   SpinLockHolder l(&heap_checker_lock);
   1515   if (!heap_checker_on) return;
   1516   size_t object_size;
   1517   if (!HaveOnHeapLocked(&ptr, &object_size)) {
   1518     RAW_LOG(ERROR, "No live heap object at %p to ignore", ptr);
   1519   } else {
   1520     RAW_VLOG(10, "Going to ignore live object at %p of %"PRIuS" bytes",
   1521                 ptr, object_size);
   1522     if (ignored_objects == NULL)  {
   1523       ignored_objects = new(Allocator::Allocate(sizeof(IgnoredObjectsMap)))
   1524                           IgnoredObjectsMap;
   1525     }
   1526     if (!ignored_objects->insert(make_pair(AsInt(ptr), object_size)).second) {
   1527       RAW_LOG(WARNING, "Object at %p is already being ignored", ptr);
   1528     }
   1529   }
   1530 }
   1531 
   1532 // static
   1533 void HeapLeakChecker::UnIgnoreObject(const void* ptr) {
   1534   SpinLockHolder l(&heap_checker_lock);
   1535   if (!heap_checker_on) return;
   1536   size_t object_size;
   1537   if (!HaveOnHeapLocked(&ptr, &object_size)) {
   1538     RAW_LOG(FATAL, "No live heap object at %p to un-ignore", ptr);
   1539   } else {
   1540     bool found = false;
   1541     if (ignored_objects) {
   1542       IgnoredObjectsMap::iterator object = ignored_objects->find(AsInt(ptr));
   1543       if (object != ignored_objects->end()  &&  object_size == object->second) {
   1544         ignored_objects->erase(object);
   1545         found = true;
   1546         RAW_VLOG(10, "Now not going to ignore live object "
   1547                     "at %p of %"PRIuS" bytes", ptr, object_size);
   1548       }
   1549     }
   1550     if (!found)  RAW_LOG(FATAL, "Object at %p has not been ignored", ptr);
   1551   }
   1552 }
   1553 
   1554 //----------------------------------------------------------------------
   1555 // HeapLeakChecker non-static functions
   1556 //----------------------------------------------------------------------
   1557 
   1558 char* HeapLeakChecker::MakeProfileNameLocked() {
   1559   RAW_DCHECK(lock_->IsHeld(), "");
   1560   RAW_DCHECK(heap_checker_lock.IsHeld(), "");
   1561   const int len = profile_name_prefix->size() + strlen(name_) + 5 +
   1562                   strlen(HeapProfileTable::kFileExt) + 1;
   1563   char* file_name = reinterpret_cast<char*>(Allocator::Allocate(len));
   1564   snprintf(file_name, len, "%s.%s-end%s",
   1565            profile_name_prefix->c_str(), name_,
   1566            HeapProfileTable::kFileExt);
   1567   return file_name;
   1568 }
   1569 
   1570 void HeapLeakChecker::Create(const char *name, bool make_start_snapshot) {
   1571   SpinLockHolder l(lock_);
   1572   name_ = NULL;  // checker is inactive
   1573   start_snapshot_ = NULL;
   1574   has_checked_ = false;
   1575   inuse_bytes_increase_ = 0;
   1576   inuse_allocs_increase_ = 0;
   1577   keep_profiles_ = false;
   1578   char* n = new char[strlen(name) + 1];   // do this before we lock
   1579   IgnoreObject(n);  // otherwise it might be treated as live due to our stack
   1580   { // Heap activity in other threads is paused for this whole scope.
   1581     SpinLockHolder al(&alignment_checker_lock);
   1582     SpinLockHolder hl(&heap_checker_lock);
   1583     MemoryRegionMap::LockHolder ml;
   1584     if (heap_checker_on  &&  profile_name_prefix != NULL) {
   1585       RAW_DCHECK(strchr(name, '/') == NULL, "must be a simple name");
   1586       memcpy(n, name, strlen(name) + 1);
   1587       name_ = n;  // checker is active
   1588       if (make_start_snapshot) {
   1589         start_snapshot_ = heap_profile->TakeSnapshot();
   1590       }
   1591 
   1592       const HeapProfileTable::Stats& t = heap_profile->total();
   1593       const size_t start_inuse_bytes = t.alloc_size - t.free_size;
   1594       const size_t start_inuse_allocs = t.allocs - t.frees;
   1595       RAW_VLOG(10, "Start check \"%s\" profile: %"PRIuS" bytes "
   1596                "in %"PRIuS" objects",
   1597                name_, start_inuse_bytes, start_inuse_allocs);
   1598     } else {
   1599       RAW_LOG(WARNING, "Heap checker is not active, "
   1600                        "hence checker \"%s\" will do nothing!", name);
   1601     RAW_LOG(WARNING, "To activate set the HEAPCHECK environment variable.\n");
   1602     }
   1603   }
   1604   if (name_ == NULL) {
   1605     UnIgnoreObject(n);
   1606     delete[] n;  // must be done after we unlock
   1607   }
   1608 }
   1609 
   1610 HeapLeakChecker::HeapLeakChecker(const char *name) : lock_(new SpinLock) {
   1611   RAW_DCHECK(strcmp(name, "_main_") != 0, "_main_ is reserved");
   1612   Create(name, true/*create start_snapshot_*/);
   1613 }
   1614 
   1615 HeapLeakChecker::HeapLeakChecker() : lock_(new SpinLock) {
   1616   if (FLAGS_heap_check_before_constructors) {
   1617     // We want to check for leaks of objects allocated during global
   1618     // constructors (i.e., objects allocated already).  So we do not
   1619     // create a baseline snapshot and hence check for leaks of objects
   1620     // that may have already been created.
   1621     Create("_main_", false);
   1622   } else {
   1623     // We want to ignore leaks of objects allocated during global
   1624     // constructors (i.e., objects allocated already).  So we snapshot
   1625     // the current heap contents and use them as a baseline that is
   1626     // not reported by the leak checker.
   1627     Create("_main_", true);
   1628   }
   1629 }
   1630 
   1631 ssize_t HeapLeakChecker::BytesLeaked() const {
   1632   SpinLockHolder l(lock_);
   1633   if (!has_checked_) {
   1634     RAW_LOG(FATAL, "*NoLeaks|SameHeap must execute before this call");
   1635   }
   1636   return inuse_bytes_increase_;
   1637 }
   1638 
   1639 ssize_t HeapLeakChecker::ObjectsLeaked() const {
   1640   SpinLockHolder l(lock_);
   1641   if (!has_checked_) {
   1642     RAW_LOG(FATAL, "*NoLeaks|SameHeap must execute before this call");
   1643   }
   1644   return inuse_allocs_increase_;
   1645 }
   1646 
   1647 // Save pid of main thread for using in naming dump files
   1648 static int32 main_thread_pid = getpid();
   1649 #ifdef HAVE_PROGRAM_INVOCATION_NAME
   1650 extern char* program_invocation_name;
   1651 extern char* program_invocation_short_name;
   1652 static const char* invocation_name() { return program_invocation_short_name; }
   1653 static string invocation_path() { return program_invocation_name; }
   1654 #else
   1655 static const char* invocation_name() { return "<your binary>"; }
   1656 static string invocation_path() { return "<your binary>"; }
   1657 #endif
   1658 
   1659 // Prints commands that users can run to get more information
   1660 // about the reported leaks.
   1661 static void SuggestPprofCommand(const char* pprof_file_arg) {
   1662   // Extra help information to print for the user when the test is
   1663   // being run in a way where the straightforward pprof command will
   1664   // not suffice.
   1665   string extra_help;
   1666 
   1667   // Common header info to print for remote runs
   1668   const string remote_header =
   1669       "This program is being executed remotely and therefore the pprof\n"
   1670       "command printed above will not work.  Either run this program\n"
   1671       "locally, or adjust the pprof command as follows to allow it to\n"
   1672       "work on your local machine:\n";
   1673 
   1674   // Extra command for fetching remote data
   1675   string fetch_cmd;
   1676 
   1677   RAW_LOG(WARNING,
   1678           "\n\n"
   1679           "If the preceding stack traces are not enough to find "
   1680           "the leaks, try running THIS shell command:\n\n"
   1681           "%s%s %s \"%s\" --inuse_objects --lines --heapcheck "
   1682           " --edgefraction=1e-10 --nodefraction=1e-10 --gv\n"
   1683           "\n"
   1684           "%s"
   1685           "If you are still puzzled about why the leaks are "
   1686           "there, try rerunning this program with "
   1687           "HEAP_CHECK_TEST_POINTER_ALIGNMENT=1 and/or with "
   1688           "HEAP_CHECK_MAX_POINTER_OFFSET=-1\n"
   1689           "If the leak report occurs in a small fraction of runs, "
   1690           "try running with TCMALLOC_MAX_FREE_QUEUE_SIZE of few hundred MB "
   1691           "or with TCMALLOC_RECLAIM_MEMORY=false, "  // only works for debugalloc
   1692           "it might help find leaks more repeatably\n",
   1693           fetch_cmd.c_str(),
   1694           "pprof",           // works as long as pprof is on your path
   1695           invocation_path().c_str(),
   1696           pprof_file_arg,
   1697           extra_help.c_str()
   1698           );
   1699 }
   1700 
   1701 bool HeapLeakChecker::DoNoLeaks(ShouldSymbolize should_symbolize) {
   1702   SpinLockHolder l(lock_);
   1703   // The locking also helps us keep the messages
   1704   // for the two checks close together.
   1705   SpinLockHolder al(&alignment_checker_lock);
   1706 
   1707   // thread-safe: protected by alignment_checker_lock
   1708   static bool have_disabled_hooks_for_symbolize = false;
   1709   // Once we've checked for leaks and symbolized the results once, it's
   1710   // not safe to do it again.  This is because in order to symbolize
   1711   // safely, we had to disable all the malloc hooks here, so we no
   1712   // longer can be confident we've collected all the data we need.
   1713   if (have_disabled_hooks_for_symbolize) {
   1714     RAW_LOG(FATAL, "Must not call heap leak checker manually after "
   1715             " program-exit's automatic check.");
   1716   }
   1717 
   1718   HeapProfileTable::Snapshot* leaks = NULL;
   1719   char* pprof_file = NULL;
   1720 
   1721   {
   1722     // Heap activity in other threads is paused during this function
   1723     // (i.e. until we got all profile difference info).
   1724     SpinLockHolder hl(&heap_checker_lock);
   1725     if (heap_checker_on == false) {
   1726       if (name_ != NULL) {  // leak checking enabled when created the checker
   1727         RAW_LOG(WARNING, "Heap leak checker got turned off after checker "
   1728                 "\"%s\" has been created, no leak check is being done for it!",
   1729                 name_);
   1730       }
   1731       return true;
   1732     }
   1733 
   1734     // Update global_region_caller_ranges. They may need to change since
   1735     // e.g. initialization because shared libraries might have been loaded or
   1736     // unloaded.
   1737     Allocator::DeleteAndNullIfNot(&global_region_caller_ranges);
   1738     ProcMapsResult pm_result = UseProcMapsLocked(DISABLE_LIBRARY_ALLOCS);
   1739     RAW_CHECK(pm_result == PROC_MAPS_USED, "");
   1740 
   1741     // Keep track of number of internally allocated objects so we
   1742     // can detect leaks in the heap-leak-checket itself
   1743     const int initial_allocs = Allocator::alloc_count();
   1744 
   1745     if (name_ == NULL) {
   1746       RAW_LOG(FATAL, "Heap leak checker must not be turned on "
   1747               "after construction of a HeapLeakChecker");
   1748     }
   1749 
   1750     MemoryRegionMap::LockHolder ml;
   1751     int a_local_var;  // Use our stack ptr to make stack data live:
   1752 
   1753     // Make the heap profile, other threads are locked out.
   1754     HeapProfileTable::Snapshot* base =
   1755         reinterpret_cast<HeapProfileTable::Snapshot*>(start_snapshot_);
   1756     RAW_DCHECK(FLAGS_heap_check_pointer_source_alignment > 0, "");
   1757     pointer_source_alignment = FLAGS_heap_check_pointer_source_alignment;
   1758     IgnoreAllLiveObjectsLocked(&a_local_var);
   1759     leaks = heap_profile->NonLiveSnapshot(base);
   1760 
   1761     inuse_bytes_increase_ = static_cast<ssize_t>(leaks->total().alloc_size);
   1762     inuse_allocs_increase_ = static_cast<ssize_t>(leaks->total().allocs);
   1763     if (leaks->Empty()) {
   1764       heap_profile->ReleaseSnapshot(leaks);
   1765       leaks = NULL;
   1766 
   1767       // We can only check for internal leaks along the no-user-leak
   1768       // path since in the leak path we temporarily release
   1769       // heap_checker_lock and another thread can come in and disturb
   1770       // allocation counts.
   1771       if (Allocator::alloc_count() != initial_allocs) {
   1772         RAW_LOG(FATAL, "Internal HeapChecker leak of %d objects ; %d -> %d",
   1773                 Allocator::alloc_count() - initial_allocs,
   1774                 initial_allocs, Allocator::alloc_count());
   1775       }
   1776     } else if (FLAGS_heap_check_test_pointer_alignment) {
   1777       if (pointer_source_alignment == 1) {
   1778         RAW_LOG(WARNING, "--heap_check_test_pointer_alignment has no effect: "
   1779                 "--heap_check_pointer_source_alignment was already set to 1");
   1780       } else {
   1781         // Try with reduced pointer aligment
   1782         pointer_source_alignment = 1;
   1783         IgnoreAllLiveObjectsLocked(&a_local_var);
   1784         HeapProfileTable::Snapshot* leaks_wo_align =
   1785             heap_profile->NonLiveSnapshot(base);
   1786         pointer_source_alignment = FLAGS_heap_check_pointer_source_alignment;
   1787         if (leaks_wo_align->Empty()) {
   1788           RAW_LOG(WARNING, "Found no leaks without pointer alignment: "
   1789                   "something might be placing pointers at "
   1790                   "unaligned addresses! This needs to be fixed.");
   1791         } else {
   1792           RAW_LOG(INFO, "Found leaks without pointer alignment as well: "
   1793                   "unaligned pointers must not be the cause of leaks.");
   1794           RAW_LOG(INFO, "--heap_check_test_pointer_alignment did not help "
   1795                   "to diagnose the leaks.");
   1796         }
   1797         heap_profile->ReleaseSnapshot(leaks_wo_align);
   1798       }
   1799     }
   1800 
   1801     if (leaks != NULL) {
   1802       pprof_file = MakeProfileNameLocked();
   1803     }
   1804   }
   1805 
   1806   has_checked_ = true;
   1807   if (leaks == NULL) {
   1808     if (FLAGS_heap_check_max_pointer_offset == -1) {
   1809       RAW_LOG(WARNING,
   1810               "Found no leaks without max_pointer_offset restriction: "
   1811               "it's possible that the default value of "
   1812               "heap_check_max_pointer_offset flag is too low. "
   1813               "Do you use pointers with larger than that offsets "
   1814               "pointing in the middle of heap-allocated objects?");
   1815     }
   1816     const HeapProfileTable::Stats& stats = heap_profile->total();
   1817     RAW_VLOG(heap_checker_info_level,
   1818              "No leaks found for check \"%s\" "
   1819              "(but no 100%% guarantee that there aren't any): "
   1820              "found %"PRId64" reachable heap objects of %"PRId64" bytes",
   1821              name_,
   1822              int64(stats.allocs - stats.frees),
   1823              int64(stats.alloc_size - stats.free_size));
   1824   } else {
   1825     if (should_symbolize == SYMBOLIZE) {
   1826       // To turn addresses into symbols, we need to fork, which is a
   1827       // problem if both parent and child end up trying to call the
   1828       // same malloc-hooks we've set up, at the same time.  To avoid
   1829       // trouble, we turn off the hooks before symbolizing.  Note that
   1830       // this makes it unsafe to ever leak-report again!  Luckily, we
   1831       // typically only want to report once in a program's run, at the
   1832       // very end.
   1833       if (MallocHook::GetNewHook() == NewHook)
   1834         MallocHook::SetNewHook(NULL);
   1835       if (MallocHook::GetDeleteHook() == DeleteHook)
   1836         MallocHook::SetDeleteHook(NULL);
   1837       MemoryRegionMap::Shutdown();
   1838       // Make sure all the hooks really got unset:
   1839       RAW_CHECK(MallocHook::GetNewHook() == NULL, "");
   1840       RAW_CHECK(MallocHook::GetDeleteHook() == NULL, "");
   1841       RAW_CHECK(MallocHook::GetMmapHook() == NULL, "");
   1842       RAW_CHECK(MallocHook::GetSbrkHook() == NULL, "");
   1843       have_disabled_hooks_for_symbolize = true;
   1844       leaks->ReportLeaks(name_, pprof_file, true);  // true = should_symbolize
   1845     } else {
   1846       leaks->ReportLeaks(name_, pprof_file, false);
   1847     }
   1848     if (FLAGS_heap_check_identify_leaks) {
   1849       leaks->ReportIndividualObjects();
   1850     }
   1851 
   1852     SuggestPprofCommand(pprof_file);
   1853 
   1854     {
   1855       SpinLockHolder hl(&heap_checker_lock);
   1856       heap_profile->ReleaseSnapshot(leaks);
   1857       Allocator::Free(pprof_file);
   1858     }
   1859   }
   1860 
   1861   return (leaks == NULL);
   1862 }
   1863 
   1864 HeapLeakChecker::~HeapLeakChecker() {
   1865   if (name_ != NULL) {  // had leak checking enabled when created the checker
   1866     if (!has_checked_) {
   1867       RAW_LOG(FATAL, "Some *NoLeaks|SameHeap method"
   1868                      " must be called on any created HeapLeakChecker");
   1869     }
   1870 
   1871     // Deallocate any snapshot taken at start
   1872     if (start_snapshot_ != NULL) {
   1873       SpinLockHolder l(&heap_checker_lock);
   1874       heap_profile->ReleaseSnapshot(
   1875           reinterpret_cast<HeapProfileTable::Snapshot*>(start_snapshot_));
   1876     }
   1877 
   1878     UnIgnoreObject(name_);
   1879     delete[] name_;
   1880     name_ = NULL;
   1881   }
   1882   delete lock_;
   1883 }
   1884 
   1885 //----------------------------------------------------------------------
   1886 // HeapLeakChecker overall heap check components
   1887 //----------------------------------------------------------------------
   1888 
   1889 // static
   1890 bool HeapLeakChecker::IsActive() {
   1891   SpinLockHolder l(&heap_checker_lock);
   1892   return heap_checker_on;
   1893 }
   1894 
   1895 vector<HeapCleaner::void_function>* HeapCleaner::heap_cleanups_ = NULL;
   1896 
   1897 // When a HeapCleaner object is intialized, add its function to the static list
   1898 // of cleaners to be run before leaks checking.
   1899 HeapCleaner::HeapCleaner(void_function f) {
   1900   if (heap_cleanups_ == NULL)
   1901     heap_cleanups_ = new vector<HeapCleaner::void_function>;
   1902   heap_cleanups_->push_back(f);
   1903 }
   1904 
   1905 // Run all of the cleanup functions and delete the vector.
   1906 void HeapCleaner::RunHeapCleanups() {
   1907   if (!heap_cleanups_)
   1908     return;
   1909   for (int i = 0; i < heap_cleanups_->size(); i++) {
   1910     void (*f)(void) = (*heap_cleanups_)[i];
   1911     f();
   1912   }
   1913   delete heap_cleanups_;
   1914   heap_cleanups_ = NULL;
   1915 }
   1916 
   1917 // Program exit heap cleanup registered as a module object destructor.
   1918 // Will not get executed when we crash on a signal.
   1919 //
   1920 void HeapLeakChecker_RunHeapCleanups() {
   1921   if (FLAGS_heap_check == "local")   // don't check heap in this mode
   1922     return;
   1923   { SpinLockHolder l(&heap_checker_lock);
   1924     // can get here (via forks?) with other pids
   1925     if (heap_checker_pid != getpid()) return;
   1926   }
   1927   HeapCleaner::RunHeapCleanups();
   1928   if (!FLAGS_heap_check_after_destructors) HeapLeakChecker::DoMainHeapCheck();
   1929 }
   1930 
   1931 static bool internal_init_start_has_run = false;
   1932 
   1933 // Called exactly once, before main() (but hopefully just before).
   1934 // This picks a good unique name for the dumped leak checking heap profiles.
   1935 //
   1936 // Because we crash when InternalInitStart is called more than once,
   1937 // it's fine that we hold heap_checker_lock only around pieces of
   1938 // this function: this is still enough for thread-safety w.r.t. other functions
   1939 // of this module.
   1940 // We can't hold heap_checker_lock throughout because it would deadlock
   1941 // on a memory allocation since our new/delete hooks can be on.
   1942 //
   1943 void HeapLeakChecker_InternalInitStart() {
   1944   { SpinLockHolder l(&heap_checker_lock);
   1945     RAW_CHECK(!internal_init_start_has_run,
   1946               "Heap-check constructor called twice.  Perhaps you both linked"
   1947               " in the heap checker, and also used LD_PRELOAD to load it?");
   1948     internal_init_start_has_run = true;
   1949 
   1950 #ifdef ADDRESS_SANITIZER
   1951     // AddressSanitizer's custom malloc conflicts with HeapChecker.
   1952     FLAGS_heap_check = "";
   1953 #endif
   1954 
   1955     if (FLAGS_heap_check.empty()) {
   1956       // turns out we do not need checking in the end; can stop profiling
   1957       HeapLeakChecker::TurnItselfOffLocked();
   1958       return;
   1959     } else if (RunningOnValgrind()) {
   1960       // There is no point in trying -- we'll just fail.
   1961       RAW_LOG(WARNING, "Can't run under Valgrind; will turn itself off");
   1962       HeapLeakChecker::TurnItselfOffLocked();
   1963       return;
   1964     }
   1965   }
   1966 
   1967   // Changing this to false can be useful when debugging heap-checker itself:
   1968   if (!FLAGS_heap_check_run_under_gdb && IsDebuggerAttached()) {
   1969     RAW_LOG(WARNING, "Someone is ptrace()ing us; will turn itself off");
   1970     SpinLockHolder l(&heap_checker_lock);
   1971     HeapLeakChecker::TurnItselfOffLocked();
   1972     return;
   1973   }
   1974 
   1975   { SpinLockHolder l(&heap_checker_lock);
   1976     if (!constructor_heap_profiling) {
   1977       RAW_LOG(FATAL, "Can not start so late. You have to enable heap checking "
   1978 	             "with HEAPCHECK=<mode>.");
   1979     }
   1980   }
   1981 
   1982   // Set all flags
   1983   RAW_DCHECK(FLAGS_heap_check_pointer_source_alignment > 0, "");
   1984   if (FLAGS_heap_check == "minimal") {
   1985     // The least we can check.
   1986     FLAGS_heap_check_before_constructors = false;  // from after main
   1987                                                    // (ignore more)
   1988     FLAGS_heap_check_after_destructors = false;  // to after cleanup
   1989                                                  // (most data is live)
   1990     FLAGS_heap_check_ignore_thread_live = true;  // ignore all live
   1991     FLAGS_heap_check_ignore_global_live = true;  // ignore all live
   1992   } else if (FLAGS_heap_check == "normal") {
   1993     // Faster than 'minimal' and not much stricter.
   1994     FLAGS_heap_check_before_constructors = true;  // from no profile (fast)
   1995     FLAGS_heap_check_after_destructors = false;  // to after cleanup
   1996                                                  // (most data is live)
   1997     FLAGS_heap_check_ignore_thread_live = true;  // ignore all live
   1998     FLAGS_heap_check_ignore_global_live = true;  // ignore all live
   1999   } else if (FLAGS_heap_check == "strict") {
   2000     // A bit stricter than 'normal': global destructors must fully clean up
   2001     // after themselves if they are present.
   2002     FLAGS_heap_check_before_constructors = true;  // from no profile (fast)
   2003     FLAGS_heap_check_after_destructors = true;  // to after destructors
   2004                                                 // (less data live)
   2005     FLAGS_heap_check_ignore_thread_live = true;  // ignore all live
   2006     FLAGS_heap_check_ignore_global_live = true;  // ignore all live
   2007   } else if (FLAGS_heap_check == "draconian") {
   2008     // Drop not very portable and not very exact live heap flooding.
   2009     FLAGS_heap_check_before_constructors = true;  // from no profile (fast)
   2010     FLAGS_heap_check_after_destructors = true;  // to after destructors
   2011                                                 // (need them)
   2012     FLAGS_heap_check_ignore_thread_live = false;  // no live flood (stricter)
   2013     FLAGS_heap_check_ignore_global_live = false;  // no live flood (stricter)
   2014   } else if (FLAGS_heap_check == "as-is") {
   2015     // do nothing: use other flags as is
   2016   } else if (FLAGS_heap_check == "local") {
   2017     // do nothing
   2018   } else {
   2019     RAW_LOG(FATAL, "Unsupported heap_check flag: %s",
   2020                    FLAGS_heap_check.c_str());
   2021   }
   2022   // FreeBSD doesn't seem to honor atexit execution order:
   2023   //    http://code.google.com/p/gperftools/issues/detail?id=375
   2024   // Since heap-checking before destructors depends on atexit running
   2025   // at the right time, on FreeBSD we always check after, even in the
   2026   // less strict modes.  This just means FreeBSD is always a bit
   2027   // stricter in its checking than other OSes.
   2028 #ifdef __FreeBSD__
   2029   FLAGS_heap_check_after_destructors = true;
   2030 #endif
   2031 
   2032   { SpinLockHolder l(&heap_checker_lock);
   2033     RAW_DCHECK(heap_checker_pid == getpid(), "");
   2034     heap_checker_on = true;
   2035     RAW_DCHECK(heap_profile, "");
   2036     HeapLeakChecker::ProcMapsResult pm_result = HeapLeakChecker::UseProcMapsLocked(HeapLeakChecker::DISABLE_LIBRARY_ALLOCS);
   2037       // might neeed to do this more than once
   2038       // if one later dynamically loads libraries that we want disabled
   2039     if (pm_result != HeapLeakChecker::PROC_MAPS_USED) {  // can't function
   2040       HeapLeakChecker::TurnItselfOffLocked();
   2041       return;
   2042     }
   2043   }
   2044 
   2045   // make a good place and name for heap profile leak dumps
   2046   string* profile_prefix =
   2047     new string(FLAGS_heap_check_dump_directory + "/" + invocation_name());
   2048 
   2049   // Finalize prefix for dumping leak checking profiles.
   2050   const int32 our_pid = getpid();   // safest to call getpid() outside lock
   2051   { SpinLockHolder l(&heap_checker_lock);
   2052     // main_thread_pid might still be 0 if this function is being called before
   2053     // global constructors.  In that case, our pid *is* the main pid.
   2054     if (main_thread_pid == 0)
   2055       main_thread_pid = our_pid;
   2056   }
   2057   char pid_buf[15];
   2058   snprintf(pid_buf, sizeof(pid_buf), ".%d", main_thread_pid);
   2059   *profile_prefix += pid_buf;
   2060   { SpinLockHolder l(&heap_checker_lock);
   2061     RAW_DCHECK(profile_name_prefix == NULL, "");
   2062     profile_name_prefix = profile_prefix;
   2063   }
   2064 
   2065   // Make sure new/delete hooks are installed properly
   2066   // and heap profiler is indeed able to keep track
   2067   // of the objects being allocated.
   2068   // We test this to make sure we are indeed checking for leaks.
   2069   char* test_str = new char[5];
   2070   size_t size;
   2071   { SpinLockHolder l(&heap_checker_lock);
   2072     RAW_CHECK(heap_profile->FindAlloc(test_str, &size),
   2073               "our own new/delete not linked?");
   2074   }
   2075   delete[] test_str;
   2076   { SpinLockHolder l(&heap_checker_lock);
   2077     // This check can fail when it should not if another thread allocates
   2078     // into this same spot right this moment,
   2079     // which is unlikely since this code runs in InitGoogle.
   2080     RAW_CHECK(!heap_profile->FindAlloc(test_str, &size),
   2081               "our own new/delete not linked?");
   2082   }
   2083   // If we crash in the above code, it probably means that
   2084   // "nm <this_binary> | grep new" will show that tcmalloc's new/delete
   2085   // implementation did not get linked-in into this binary
   2086   // (i.e. nm will list __builtin_new and __builtin_vec_new as undefined).
   2087   // If this happens, it is a BUILD bug to be fixed.
   2088 
   2089   RAW_VLOG(heap_checker_info_level,
   2090            "WARNING: Perftools heap leak checker is active "
   2091            "-- Performance may suffer");
   2092 
   2093   if (FLAGS_heap_check != "local") {
   2094     HeapLeakChecker* main_hc = new HeapLeakChecker();
   2095     SpinLockHolder l(&heap_checker_lock);
   2096     RAW_DCHECK(main_heap_checker == NULL,
   2097                "Repeated creation of main_heap_checker");
   2098     main_heap_checker = main_hc;
   2099     do_main_heap_check = true;
   2100   }
   2101 
   2102   { SpinLockHolder l(&heap_checker_lock);
   2103     RAW_CHECK(heap_checker_on  &&  constructor_heap_profiling,
   2104               "Leak checking is expected to be fully turned on now");
   2105   }
   2106 
   2107   // For binaries built in debug mode, this will set release queue of
   2108   // debugallocation.cc to 100M to make it less likely for real leaks to
   2109   // be hidden due to reuse of heap memory object addresses.
   2110   // Running a test with --malloc_reclaim_memory=0 would help find leaks even
   2111   // better, but the test might run out of memory as a result.
   2112   // The scenario is that a heap object at address X is allocated and freed,
   2113   // but some other data-structure still retains a pointer to X.
   2114   // Then the same heap memory is used for another object, which is leaked,
   2115   // but the leak is not noticed due to the pointer to the original object at X.
   2116   // TODO(csilvers): support this in some manner.
   2117 #if 0
   2118   SetCommandLineOptionWithMode("max_free_queue_size", "104857600",  // 100M
   2119                                SET_FLAG_IF_DEFAULT);
   2120 #endif
   2121 }
   2122 
   2123 // We want this to run early as well, but not so early as
   2124 // ::BeforeConstructors (we want flag assignments to have already
   2125 // happened, for instance).  Initializer-registration does the trick.
   2126 REGISTER_MODULE_INITIALIZER(init_start, HeapLeakChecker_InternalInitStart());
   2127 REGISTER_MODULE_DESTRUCTOR(init_start, HeapLeakChecker_RunHeapCleanups());
   2128 
   2129 // static
   2130 bool HeapLeakChecker::NoGlobalLeaksMaybeSymbolize(
   2131     ShouldSymbolize should_symbolize) {
   2132   // we never delete or change main_heap_checker once it's set:
   2133   HeapLeakChecker* main_hc = GlobalChecker();
   2134   if (main_hc) {
   2135     RAW_VLOG(10, "Checking for whole-program memory leaks");
   2136     return main_hc->DoNoLeaks(should_symbolize);
   2137   }
   2138   return true;
   2139 }
   2140 
   2141 // static
   2142 bool HeapLeakChecker::DoMainHeapCheck() {
   2143   if (FLAGS_heap_check_delay_seconds > 0) {
   2144     sleep(FLAGS_heap_check_delay_seconds);
   2145   }
   2146   { SpinLockHolder l(&heap_checker_lock);
   2147     if (!do_main_heap_check) return false;
   2148     RAW_DCHECK(heap_checker_pid == getpid(), "");
   2149     do_main_heap_check = false;  // will do it now; no need to do it more
   2150   }
   2151 
   2152   // The program is over, so it's safe to symbolize addresses (which
   2153   // requires a fork) because no serious work is expected to be done
   2154   // after this.  Symbolizing is really useful -- knowing what
   2155   // function has a leak is better than knowing just an address --
   2156   // and while we can only safely symbolize once in a program run,
   2157   // now is the time (after all, there's no "later" that would be better).
   2158   if (!NoGlobalLeaksMaybeSymbolize(SYMBOLIZE)) {
   2159     if (FLAGS_heap_check_identify_leaks) {
   2160       RAW_LOG(FATAL, "Whole-program memory leaks found.");
   2161     }
   2162     RAW_LOG(ERROR, "Exiting with error code (instead of crashing) "
   2163                    "because of whole-program memory leaks");
   2164     // We don't want to call atexit() routines!
   2165     _exit(FLAGS_heap_check_error_exit_code);
   2166   }
   2167   return true;
   2168 }
   2169 
   2170 // static
   2171 HeapLeakChecker* HeapLeakChecker::GlobalChecker() {
   2172   SpinLockHolder l(&heap_checker_lock);
   2173   return main_heap_checker;
   2174 }
   2175 
   2176 // static
   2177 bool HeapLeakChecker::NoGlobalLeaks() {
   2178   // symbolizing requires a fork, which isn't safe to do in general.
   2179   return NoGlobalLeaksMaybeSymbolize(DO_NOT_SYMBOLIZE);
   2180 }
   2181 
   2182 // static
   2183 void HeapLeakChecker::CancelGlobalCheck() {
   2184   SpinLockHolder l(&heap_checker_lock);
   2185   if (do_main_heap_check) {
   2186     RAW_VLOG(heap_checker_info_level,
   2187              "Canceling the automatic at-exit whole-program memory leak check");
   2188     do_main_heap_check = false;
   2189   }
   2190 }
   2191 
   2192 // static
   2193 void HeapLeakChecker::BeforeConstructorsLocked() {
   2194   RAW_DCHECK(heap_checker_lock.IsHeld(), "");
   2195   RAW_CHECK(!constructor_heap_profiling,
   2196             "BeforeConstructorsLocked called multiple times");
   2197 #ifdef ADDRESS_SANITIZER
   2198   // AddressSanitizer's custom malloc conflicts with HeapChecker.
   2199   return;
   2200 #endif
   2201   // Set hooks early to crash if 'new' gets called before we make heap_profile,
   2202   // and make sure no other hooks existed:
   2203   RAW_CHECK(MallocHook::AddNewHook(&NewHook), "");
   2204   RAW_CHECK(MallocHook::AddDeleteHook(&DeleteHook), "");
   2205   constructor_heap_profiling = true;
   2206   MemoryRegionMap::Init(1, /* use_buckets */ false);
   2207     // Set up MemoryRegionMap with (at least) one caller stack frame to record
   2208     // (important that it's done before HeapProfileTable creation below).
   2209   Allocator::Init();
   2210   RAW_CHECK(heap_profile == NULL, "");
   2211   heap_profile = new(Allocator::Allocate(sizeof(HeapProfileTable)))
   2212       HeapProfileTable(&Allocator::Allocate, &Allocator::Free,
   2213                        /* profile_mmap */ false);
   2214   RAW_VLOG(10, "Starting tracking the heap");
   2215   heap_checker_on = true;
   2216 }
   2217 
   2218 // static
   2219 void HeapLeakChecker::TurnItselfOffLocked() {
   2220   RAW_DCHECK(heap_checker_lock.IsHeld(), "");
   2221   // Set FLAGS_heap_check to "", for users who test for it
   2222   if (!FLAGS_heap_check.empty())  // be a noop in the common case
   2223     FLAGS_heap_check.clear();     // because clear() could allocate memory
   2224   if (constructor_heap_profiling) {
   2225     RAW_CHECK(heap_checker_on, "");
   2226     RAW_VLOG(heap_checker_info_level, "Turning perftools heap leak checking off");
   2227     heap_checker_on = false;
   2228     // Unset our hooks checking they were set:
   2229     RAW_CHECK(MallocHook::RemoveNewHook(&NewHook), "");
   2230     RAW_CHECK(MallocHook::RemoveDeleteHook(&DeleteHook), "");
   2231     Allocator::DeleteAndNull(&heap_profile);
   2232     // free our optional global data:
   2233     Allocator::DeleteAndNullIfNot(&ignored_objects);
   2234     Allocator::DeleteAndNullIfNot(&disabled_ranges);
   2235     Allocator::DeleteAndNullIfNot(&global_region_caller_ranges);
   2236     Allocator::Shutdown();
   2237     MemoryRegionMap::Shutdown();
   2238   }
   2239   RAW_CHECK(!heap_checker_on, "");
   2240 }
   2241 
   2242 extern bool heap_leak_checker_bcad_variable;  // in heap-checker-bcad.cc
   2243 
   2244 static bool has_called_before_constructors = false;
   2245 
   2246 // TODO(maxim): inline this function with
   2247 // MallocHook_InitAtFirstAllocation_HeapLeakChecker, and also rename
   2248 // HeapLeakChecker::BeforeConstructorsLocked.
   2249 void HeapLeakChecker_BeforeConstructors() {
   2250   SpinLockHolder l(&heap_checker_lock);
   2251   // We can be called from several places: the first mmap/sbrk/alloc call
   2252   // or the first global c-tor from heap-checker-bcad.cc:
   2253   // Do not re-execute initialization:
   2254   if (has_called_before_constructors) return;
   2255   has_called_before_constructors = true;
   2256 
   2257   heap_checker_pid = getpid();  // set it always
   2258   heap_leak_checker_bcad_variable = true;
   2259   // just to reference it, so that heap-checker-bcad.o is linked in
   2260 
   2261   // This function can be called *very* early, before the normal
   2262   // global-constructor that sets FLAGS_verbose.  Set it manually now,
   2263   // so the RAW_LOG messages here are controllable.
   2264   const char* verbose_str = GetenvBeforeMain("PERFTOOLS_VERBOSE");
   2265   if (verbose_str && atoi(verbose_str)) {  // different than the default of 0?
   2266     FLAGS_verbose = atoi(verbose_str);
   2267   }
   2268 
   2269   bool need_heap_check = true;
   2270   // The user indicates a desire for heap-checking via the HEAPCHECK
   2271   // environment variable.  If it's not set, there's no way to do
   2272   // heap-checking.
   2273   if (!GetenvBeforeMain("HEAPCHECK")) {
   2274     need_heap_check = false;
   2275   }
   2276 #ifdef HAVE_GETEUID
   2277   if (need_heap_check && getuid() != geteuid()) {
   2278     // heap-checker writes out files.  Thus, for security reasons, we don't
   2279     // recognize the env. var. to turn on heap-checking if we're setuid.
   2280     RAW_LOG(WARNING, ("HeapChecker: ignoring HEAPCHECK because "
   2281                       "program seems to be setuid\n"));
   2282     need_heap_check = false;
   2283   }
   2284 #endif
   2285   if (need_heap_check) {
   2286     HeapLeakChecker::BeforeConstructorsLocked();
   2287   }
   2288 }
   2289 
   2290 // This function overrides the weak function defined in malloc_hook.cc and
   2291 // called by one of the initial malloc hooks (malloc_hook.cc) when the very
   2292 // first memory allocation or an mmap/sbrk happens.  This ensures that
   2293 // HeapLeakChecker is initialized and installs all its hooks early enough to
   2294 // track absolutely all memory allocations and all memory region acquisitions
   2295 // via mmap and sbrk.
   2296 extern "C" void MallocHook_InitAtFirstAllocation_HeapLeakChecker() {
   2297   HeapLeakChecker_BeforeConstructors();
   2298 }
   2299 
   2300 // This function is executed after all global object destructors run.
   2301 void HeapLeakChecker_AfterDestructors() {
   2302   { SpinLockHolder l(&heap_checker_lock);
   2303     // can get here (via forks?) with other pids
   2304     if (heap_checker_pid != getpid()) return;
   2305   }
   2306   if (FLAGS_heap_check_after_destructors) {
   2307     if (HeapLeakChecker::DoMainHeapCheck()) {
   2308       const struct timespec sleep_time = { 0, 500000000 };  // 500 ms
   2309       nanosleep(&sleep_time, NULL);
   2310         // Need this hack to wait for other pthreads to exit.
   2311         // Otherwise tcmalloc find errors
   2312         // on a free() call from pthreads.
   2313     }
   2314   }
   2315   SpinLockHolder l(&heap_checker_lock);
   2316   RAW_CHECK(!do_main_heap_check, "should have done it");
   2317 }
   2318 
   2319 //----------------------------------------------------------------------
   2320 // HeapLeakChecker disabling helpers
   2321 //----------------------------------------------------------------------
   2322 
   2323 // These functions are at the end of the file to prevent their inlining:
   2324 
   2325 // static
   2326 void HeapLeakChecker::DisableChecksFromToLocked(const void* start_address,
   2327                                                 const void* end_address,
   2328                                                 int max_depth) {
   2329   RAW_DCHECK(heap_checker_lock.IsHeld(), "");
   2330   RAW_DCHECK(start_address < end_address, "");
   2331   if (disabled_ranges == NULL) {
   2332     disabled_ranges = new(Allocator::Allocate(sizeof(DisabledRangeMap)))
   2333                         DisabledRangeMap;
   2334   }
   2335   RangeValue value;
   2336   value.start_address = AsInt(start_address);
   2337   value.max_depth = max_depth;
   2338   if (disabled_ranges->insert(make_pair(AsInt(end_address), value)).second) {
   2339     RAW_VLOG(10, "Disabling leak checking in stack traces "
   2340                 "under frame addresses between %p..%p",
   2341                 start_address, end_address);
   2342   } else {  // check that this is just a verbatim repetition
   2343     RangeValue const& val = disabled_ranges->find(AsInt(end_address))->second;
   2344     if (val.max_depth != value.max_depth  ||
   2345         val.start_address != value.start_address) {
   2346       RAW_LOG(FATAL, "Two DisableChecksToHereFrom calls conflict: "
   2347                      "(%p, %p, %d) vs. (%p, %p, %d)",
   2348                      AsPtr(val.start_address), end_address, val.max_depth,
   2349                      start_address, end_address, max_depth);
   2350     }
   2351   }
   2352 }
   2353 
   2354 // static
   2355 inline bool HeapLeakChecker::HaveOnHeapLocked(const void** ptr,
   2356                                               size_t* object_size) {
   2357   // Commented-out because HaveOnHeapLocked is very performance-critical:
   2358   // RAW_DCHECK(heap_checker_lock.IsHeld(), "");
   2359   const uintptr_t addr = AsInt(*ptr);
   2360   if (heap_profile->FindInsideAlloc(
   2361         *ptr, max_heap_object_size, ptr, object_size)) {
   2362     RAW_VLOG(16, "Got pointer into %p at +%"PRIuPTR" offset",
   2363              *ptr, addr - AsInt(*ptr));
   2364     return true;
   2365   }
   2366   return false;
   2367 }
   2368 
   2369 // static
   2370 const void* HeapLeakChecker::GetAllocCaller(void* ptr) {
   2371   // this is used only in the unittest, so the heavy checks are fine
   2372   HeapProfileTable::AllocInfo info;
   2373   { SpinLockHolder l(&heap_checker_lock);
   2374     RAW_CHECK(heap_profile->FindAllocDetails(ptr, &info), "");
   2375   }
   2376   RAW_CHECK(info.stack_depth >= 1, "");
   2377   return info.call_stack[0];
   2378 }
   2379