1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "base/allocator/allocator_shim.h" 6 7 #include <config.h> 8 #include "base/allocator/allocator_extension_thunks.h" 9 #include "base/profiler/alternate_timer.h" 10 #include "base/sysinfo.h" 11 #include "jemalloc.h" 12 13 // When defined, different heap allocators can be used via an environment 14 // variable set before running the program. This may reduce the amount 15 // of inlining that we get with malloc/free/etc. Disabling makes it 16 // so that only tcmalloc can be used. 17 #define ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 18 19 // TODO(mbelshe): Ensure that all calls to tcmalloc have the proper call depth 20 // from the "user code" so that debugging tools (HeapChecker) can work. 21 22 // __THROW is defined in glibc systems. It means, counter-intuitively, 23 // "This function will never throw an exception." It's an optional 24 // optimization tool, but we may need to use it to match glibc prototypes. 25 #ifndef __THROW // I guess we're not on a glibc system 26 # define __THROW // __THROW is just an optimization, so ok to make it "" 27 #endif 28 29 // new_mode behaves similarly to MSVC's _set_new_mode. 30 // If flag is 0 (default), calls to malloc will behave normally. 31 // If flag is 1, calls to malloc will behave like calls to new, 32 // and the std_new_handler will be invoked on failure. 33 // Can be set by calling _set_new_mode(). 34 static int new_mode = 0; 35 36 typedef enum { 37 TCMALLOC, // TCMalloc is the default allocator. 38 JEMALLOC, // JEMalloc. 39 WINHEAP, // Windows Heap (standard Windows allocator). 40 WINLFH, // Windows LFH Heap. 41 } Allocator; 42 43 // This is the default allocator. This value can be changed at startup by 44 // specifying environment variables shown below it. 45 // See SetupSubprocessAllocator() to specify a default secondary (subprocess) 46 // allocator. 47 // TODO(jar): Switch to using TCMALLOC for the renderer as well. 48 #if (defined(ADDRESS_SANITIZER) && defined(OS_WIN)) 49 // The Windows implementation of Asan requires the use of "WINHEAP". 50 static Allocator allocator = WINHEAP; 51 #else 52 static Allocator allocator = TCMALLOC; 53 #endif 54 // The names of the environment variables that can optionally control the 55 // selection of the allocator. The primary may be used to control overall 56 // allocator selection, and the secondary can be used to specify an allocator 57 // to use in sub-processes. 58 static const char primary_name[] = "CHROME_ALLOCATOR"; 59 static const char secondary_name[] = "CHROME_ALLOCATOR_2"; 60 61 // We include tcmalloc and the win_allocator to get as much inlining as 62 // possible. 63 #include "debugallocation_shim.cc" 64 #include "win_allocator.cc" 65 66 // Forward declarations from jemalloc. 67 extern "C" { 68 void* je_malloc(size_t s); 69 void* je_realloc(void* p, size_t s); 70 void je_free(void* s); 71 size_t je_msize(void* p); 72 bool je_malloc_init_hard(); 73 void* je_memalign(size_t a, size_t s); 74 } 75 76 // Call the new handler, if one has been set. 77 // Returns true on successfully calling the handler, false otherwise. 78 inline bool call_new_handler(bool nothrow) { 79 // Get the current new handler. NB: this function is not 80 // thread-safe. We make a feeble stab at making it so here, but 81 // this lock only protects against tcmalloc interfering with 82 // itself, not with other libraries calling set_new_handler. 83 std::new_handler nh; 84 { 85 SpinLockHolder h(&set_new_handler_lock); 86 nh = std::set_new_handler(0); 87 (void) std::set_new_handler(nh); 88 } 89 #if (defined(__GNUC__) && !defined(__EXCEPTIONS)) || \ 90 (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS) 91 if (!nh) 92 return false; 93 // Since exceptions are disabled, we don't really know if new_handler 94 // failed. Assume it will abort if it fails. 95 (*nh)(); 96 return false; // break out of the retry loop. 97 #else 98 // If no new_handler is established, the allocation failed. 99 if (!nh) { 100 if (nothrow) 101 return false; 102 throw std::bad_alloc(); 103 } 104 // Otherwise, try the new_handler. If it returns, retry the 105 // allocation. If it throws std::bad_alloc, fail the allocation. 106 // if it throws something else, don't interfere. 107 try { 108 (*nh)(); 109 } catch (const std::bad_alloc&) { 110 if (!nothrow) 111 throw; 112 return true; 113 } 114 #endif // (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS) 115 return false; 116 } 117 118 extern "C" { 119 void* malloc(size_t size) __THROW { 120 void* ptr; 121 for (;;) { 122 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 123 switch (allocator) { 124 case JEMALLOC: 125 ptr = je_malloc(size); 126 break; 127 case WINHEAP: 128 case WINLFH: 129 ptr = win_heap_malloc(size); 130 break; 131 case TCMALLOC: 132 default: 133 ptr = do_malloc(size); 134 break; 135 } 136 #else 137 // TCMalloc case. 138 ptr = do_malloc(size); 139 #endif 140 if (ptr) 141 return ptr; 142 143 if (!new_mode || !call_new_handler(true)) 144 break; 145 } 146 return ptr; 147 } 148 149 void free(void* p) __THROW { 150 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 151 switch (allocator) { 152 case JEMALLOC: 153 je_free(p); 154 return; 155 case WINHEAP: 156 case WINLFH: 157 win_heap_free(p); 158 return; 159 } 160 #endif 161 // TCMalloc case. 162 do_free(p); 163 } 164 165 void* realloc(void* ptr, size_t size) __THROW { 166 // Webkit is brittle for allocators that return NULL for malloc(0). The 167 // realloc(0, 0) code path does not guarantee a non-NULL return, so be sure 168 // to call malloc for this case. 169 if (!ptr) 170 return malloc(size); 171 172 void* new_ptr; 173 for (;;) { 174 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 175 switch (allocator) { 176 case JEMALLOC: 177 new_ptr = je_realloc(ptr, size); 178 break; 179 case WINHEAP: 180 case WINLFH: 181 new_ptr = win_heap_realloc(ptr, size); 182 break; 183 case TCMALLOC: 184 default: 185 new_ptr = do_realloc(ptr, size); 186 break; 187 } 188 #else 189 // TCMalloc case. 190 new_ptr = do_realloc(ptr, size); 191 #endif 192 193 // Subtle warning: NULL return does not alwas indicate out-of-memory. If 194 // the requested new size is zero, realloc should free the ptr and return 195 // NULL. 196 if (new_ptr || !size) 197 return new_ptr; 198 if (!new_mode || !call_new_handler(true)) 199 break; 200 } 201 return new_ptr; 202 } 203 204 // TODO(mbelshe): Implement this for other allocators. 205 void malloc_stats(void) __THROW { 206 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 207 switch (allocator) { 208 case JEMALLOC: 209 // No stats. 210 return; 211 case WINHEAP: 212 case WINLFH: 213 // No stats. 214 return; 215 } 216 #endif 217 tc_malloc_stats(); 218 } 219 220 #ifdef WIN32 221 222 extern "C" size_t _msize(void* p) { 223 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 224 switch (allocator) { 225 case JEMALLOC: 226 return je_msize(p); 227 case WINHEAP: 228 case WINLFH: 229 return win_heap_msize(p); 230 } 231 #endif 232 return MallocExtension::instance()->GetAllocatedSize(p); 233 } 234 235 // This is included to resolve references from libcmt. 236 extern "C" intptr_t _get_heap_handle() { 237 return 0; 238 } 239 240 static bool get_allocator_waste_size_thunk(size_t* size) { 241 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 242 switch (allocator) { 243 case JEMALLOC: 244 case WINHEAP: 245 case WINLFH: 246 // TODO(alexeif): Implement for allocators other than tcmalloc. 247 return false; 248 } 249 #endif 250 size_t heap_size, allocated_bytes, unmapped_bytes; 251 MallocExtension* ext = MallocExtension::instance(); 252 if (ext->GetNumericProperty("generic.heap_size", &heap_size) && 253 ext->GetNumericProperty("generic.current_allocated_bytes", 254 &allocated_bytes) && 255 ext->GetNumericProperty("tcmalloc.pageheap_unmapped_bytes", 256 &unmapped_bytes)) { 257 *size = heap_size - allocated_bytes - unmapped_bytes; 258 return true; 259 } 260 return false; 261 } 262 263 static void get_stats_thunk(char* buffer, int buffer_length) { 264 MallocExtension::instance()->GetStats(buffer, buffer_length); 265 } 266 267 static void release_free_memory_thunk() { 268 MallocExtension::instance()->ReleaseFreeMemory(); 269 } 270 271 // The CRT heap initialization stub. 272 extern "C" int _heap_init() { 273 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 274 // Don't use the environment variable if ADDRESS_SANITIZER is defined on 275 // Windows, as the implementation requires Winheap to be the allocator. 276 #if !(defined(ADDRESS_SANITIZER) && defined(OS_WIN)) 277 const char* environment_value = GetenvBeforeMain(primary_name); 278 if (environment_value) { 279 if (!stricmp(environment_value, "jemalloc")) 280 allocator = JEMALLOC; 281 else if (!stricmp(environment_value, "winheap")) 282 allocator = WINHEAP; 283 else if (!stricmp(environment_value, "winlfh")) 284 allocator = WINLFH; 285 else if (!stricmp(environment_value, "tcmalloc")) 286 allocator = TCMALLOC; 287 } 288 #endif 289 290 switch (allocator) { 291 case JEMALLOC: 292 return je_malloc_init_hard() ? 0 : 1; 293 case WINHEAP: 294 return win_heap_init(false) ? 1 : 0; 295 case WINLFH: 296 return win_heap_init(true) ? 1 : 0; 297 case TCMALLOC: 298 default: 299 // fall through 300 break; 301 } 302 #endif 303 // Initializing tcmalloc. 304 // We intentionally leak this object. It lasts for the process 305 // lifetime. Trying to teardown at _heap_term() is so late that 306 // you can't do anything useful anyway. 307 new TCMallocGuard(); 308 309 // Provide optional hook for monitoring allocation quantities on a per-thread 310 // basis. Only set the hook if the environment indicates this needs to be 311 // enabled. 312 const char* profiling = 313 GetenvBeforeMain(tracked_objects::kAlternateProfilerTime); 314 if (profiling && *profiling == '1') { 315 tracked_objects::SetAlternateTimeSource( 316 tcmalloc::ThreadCache::GetBytesAllocatedOnCurrentThread, 317 tracked_objects::TIME_SOURCE_TYPE_TCMALLOC); 318 } 319 320 base::allocator::thunks::SetGetAllocatorWasteSizeFunction( 321 get_allocator_waste_size_thunk); 322 base::allocator::thunks::SetGetStatsFunction(get_stats_thunk); 323 base::allocator::thunks::SetReleaseFreeMemoryFunction( 324 release_free_memory_thunk); 325 326 return 1; 327 } 328 329 // The CRT heap cleanup stub. 330 extern "C" void _heap_term() {} 331 332 // We set this to 1 because part of the CRT uses a check of _crtheap != 0 333 // to test whether the CRT has been initialized. Once we've ripped out 334 // the allocators from libcmt, we need to provide this definition so that 335 // the rest of the CRT is still usable. 336 extern "C" void* _crtheap = reinterpret_cast<void*>(1); 337 338 // Provide support for aligned memory through Windows only _aligned_malloc(). 339 void* _aligned_malloc(size_t size, size_t alignment) { 340 // _aligned_malloc guarantees parameter validation, so do so here. These 341 // checks are somewhat stricter than _aligned_malloc() since we're effectively 342 // using memalign() under the hood. 343 DCHECK_GT(size, 0U); 344 DCHECK_EQ(alignment & (alignment - 1), 0U); 345 DCHECK_EQ(alignment % sizeof(void*), 0U); 346 347 void* ptr; 348 for (;;) { 349 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 350 switch (allocator) { 351 case JEMALLOC: 352 ptr = je_memalign(alignment, size); 353 break; 354 case WINHEAP: 355 case WINLFH: 356 ptr = win_heap_memalign(alignment, size); 357 break; 358 case TCMALLOC: 359 default: 360 ptr = tc_memalign(alignment, size); 361 break; 362 } 363 #else 364 // TCMalloc case. 365 ptr = tc_memalign(alignment, size); 366 #endif 367 if (ptr) { 368 // Sanity check alignment. 369 DCHECK_EQ(reinterpret_cast<uintptr_t>(ptr) & (alignment - 1), 0U); 370 return ptr; 371 } 372 373 if (!new_mode || !call_new_handler(true)) 374 break; 375 } 376 return ptr; 377 } 378 379 void _aligned_free(void* p) { 380 // Both JEMalloc and TCMalloc return pointers from memalign() that are safe to 381 // use with free(). Pointers allocated with win_heap_memalign() MUST be freed 382 // via win_heap_memalign_free() since the aligned pointer is not the real one. 383 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 384 switch (allocator) { 385 case JEMALLOC: 386 je_free(p); 387 return; 388 case WINHEAP: 389 case WINLFH: 390 win_heap_memalign_free(p); 391 return; 392 } 393 #endif 394 // TCMalloc case. 395 do_free(p); 396 } 397 398 #endif // WIN32 399 400 #include "generic_allocators.cc" 401 402 } // extern C 403 404 namespace base { 405 namespace allocator { 406 407 void SetupSubprocessAllocator() { 408 #ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 409 size_t primary_length = 0; 410 getenv_s(&primary_length, NULL, 0, primary_name); 411 412 size_t secondary_length = 0; 413 char buffer[20]; 414 getenv_s(&secondary_length, buffer, sizeof(buffer), secondary_name); 415 DCHECK_GT(sizeof(buffer), secondary_length); 416 buffer[sizeof(buffer) - 1] = '\0'; 417 418 if (secondary_length || !primary_length) { 419 // Don't use the environment variable if ADDRESS_SANITIZER is defined on 420 // Windows, as the implementation require Winheap to be the allocator. 421 #if !(defined(ADDRESS_SANITIZER) && defined(OS_WIN)) 422 const char* secondary_value = secondary_length ? buffer : "TCMALLOC"; 423 // Force renderer (or other subprocesses) to use secondary_value. 424 #else 425 const char* secondary_value = "WINHEAP"; 426 #endif 427 int ret_val = _putenv_s(primary_name, secondary_value); 428 DCHECK_EQ(0, ret_val); 429 } 430 #endif // ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 431 } 432 433 void* TCMallocDoMallocForTest(size_t size) { 434 return do_malloc(size); 435 } 436 437 void TCMallocDoFreeForTest(void* ptr) { 438 do_free(ptr); 439 } 440 441 size_t ExcludeSpaceForMarkForTest(size_t size) { 442 return ExcludeSpaceForMark(size); 443 } 444 445 } // namespace allocator. 446 } // namespace base. 447
