1 #define JEMALLOC_C_ 2 #include "jemalloc/internal/jemalloc_preamble.h" 3 #include "jemalloc/internal/jemalloc_internal_includes.h" 4 5 #include "jemalloc/internal/assert.h" 6 #include "jemalloc/internal/atomic.h" 7 #include "jemalloc/internal/ctl.h" 8 #include "jemalloc/internal/extent_dss.h" 9 #include "jemalloc/internal/extent_mmap.h" 10 #include "jemalloc/internal/jemalloc_internal_types.h" 11 #include "jemalloc/internal/log.h" 12 #include "jemalloc/internal/malloc_io.h" 13 #include "jemalloc/internal/mutex.h" 14 #include "jemalloc/internal/rtree.h" 15 #include "jemalloc/internal/size_classes.h" 16 #include "jemalloc/internal/spin.h" 17 #include "jemalloc/internal/sz.h" 18 #include "jemalloc/internal/ticker.h" 19 #include "jemalloc/internal/util.h" 20 21 /******************************************************************************/ 22 /* Data. */ 23 24 /* Runtime configuration options. */ 25 const char *je_malloc_conf 26 #ifndef _WIN32 27 JEMALLOC_ATTR(weak) 28 #endif 29 ; 30 bool opt_abort = 31 #ifdef JEMALLOC_DEBUG 32 true 33 #else 34 false 35 #endif 36 ; 37 bool opt_abort_conf = 38 #ifdef JEMALLOC_DEBUG 39 true 40 #else 41 false 42 #endif 43 ; 44 const char *opt_junk = 45 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 46 "true" 47 #else 48 "false" 49 #endif 50 ; 51 bool opt_junk_alloc = 52 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 53 true 54 #else 55 false 56 #endif 57 ; 58 bool opt_junk_free = 59 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 60 true 61 #else 62 false 63 #endif 64 ; 65 66 bool opt_utrace = false; 67 bool opt_xmalloc = false; 68 bool opt_zero = false; 69 unsigned opt_narenas = 0; 70 71 unsigned ncpus; 72 73 /* Protects arenas initialization. */ 74 malloc_mutex_t arenas_lock; 75 /* 76 * Arenas that are used to service external requests. Not all elements of the 77 * arenas array are necessarily used; arenas are created lazily as needed. 78 * 79 * arenas[0..narenas_auto) are used for automatic multiplexing of threads and 80 * arenas. arenas[narenas_auto..narenas_total) are only used if the application 81 * takes some action to create them and allocate from them. 82 * 83 * Points to an arena_t. 84 */ 85 JEMALLOC_ALIGNED(CACHELINE) 86 atomic_p_t arenas[MALLOCX_ARENA_LIMIT]; 87 static atomic_u_t narenas_total; /* Use narenas_total_*(). */ 88 static arena_t *a0; /* arenas[0]; read-only after initialization. */ 89 unsigned narenas_auto; /* Read-only after initialization. */ 90 91 typedef enum { 92 malloc_init_uninitialized = 3, 93 malloc_init_a0_initialized = 2, 94 malloc_init_recursible = 1, 95 malloc_init_initialized = 0 /* Common case --> jnz. */ 96 } malloc_init_t; 97 static malloc_init_t malloc_init_state = malloc_init_uninitialized; 98 99 /* False should be the common case. Set to true to trigger initialization. */ 100 bool malloc_slow = true; 101 102 /* When malloc_slow is true, set the corresponding bits for sanity check. */ 103 enum { 104 flag_opt_junk_alloc = (1U), 105 flag_opt_junk_free = (1U << 1), 106 flag_opt_zero = (1U << 2), 107 flag_opt_utrace = (1U << 3), 108 flag_opt_xmalloc = (1U << 4) 109 }; 110 static uint8_t malloc_slow_flags; 111 112 #ifdef JEMALLOC_THREADED_INIT 113 /* Used to let the initializing thread recursively allocate. */ 114 # define NO_INITIALIZER ((unsigned long)0) 115 # define INITIALIZER pthread_self() 116 # define IS_INITIALIZER (malloc_initializer == pthread_self()) 117 static pthread_t malloc_initializer = NO_INITIALIZER; 118 #else 119 # define NO_INITIALIZER false 120 # define INITIALIZER true 121 # define IS_INITIALIZER malloc_initializer 122 static bool malloc_initializer = NO_INITIALIZER; 123 #endif 124 125 /* Used to avoid initialization races. */ 126 #ifdef _WIN32 127 #if _WIN32_WINNT >= 0x0600 128 static malloc_mutex_t init_lock = SRWLOCK_INIT; 129 #else 130 static malloc_mutex_t init_lock; 131 static bool init_lock_initialized = false; 132 133 JEMALLOC_ATTR(constructor) 134 static void WINAPI 135 _init_init_lock(void) { 136 /* 137 * If another constructor in the same binary is using mallctl to e.g. 138 * set up extent hooks, it may end up running before this one, and 139 * malloc_init_hard will crash trying to lock the uninitialized lock. So 140 * we force an initialization of the lock in malloc_init_hard as well. 141 * We don't try to care about atomicity of the accessed to the 142 * init_lock_initialized boolean, since it really only matters early in 143 * the process creation, before any separate thread normally starts 144 * doing anything. 145 */ 146 if (!init_lock_initialized) { 147 malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT, 148 malloc_mutex_rank_exclusive); 149 } 150 init_lock_initialized = true; 151 } 152 153 #ifdef _MSC_VER 154 # pragma section(".CRT$XCU", read) 155 JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used) 156 static const void (WINAPI *init_init_lock)(void) = _init_init_lock; 157 #endif 158 #endif 159 #else 160 static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; 161 #endif 162 163 typedef struct { 164 void *p; /* Input pointer (as in realloc(p, s)). */ 165 size_t s; /* Request size. */ 166 void *r; /* Result pointer. */ 167 } malloc_utrace_t; 168 169 #ifdef JEMALLOC_UTRACE 170 # define UTRACE(a, b, c) do { \ 171 if (unlikely(opt_utrace)) { \ 172 int utrace_serrno = errno; \ 173 malloc_utrace_t ut; \ 174 ut.p = (a); \ 175 ut.s = (b); \ 176 ut.r = (c); \ 177 utrace(&ut, sizeof(ut)); \ 178 errno = utrace_serrno; \ 179 } \ 180 } while (0) 181 #else 182 # define UTRACE(a, b, c) 183 #endif 184 185 /* Whether encountered any invalid config options. */ 186 static bool had_conf_error = false; 187 188 /******************************************************************************/ 189 /* 190 * Function prototypes for static functions that are referenced prior to 191 * definition. 192 */ 193 194 static bool malloc_init_hard_a0(void); 195 static bool malloc_init_hard(void); 196 197 /******************************************************************************/ 198 /* 199 * Begin miscellaneous support functions. 200 */ 201 202 bool 203 malloc_initialized(void) { 204 return (malloc_init_state == malloc_init_initialized); 205 } 206 207 JEMALLOC_ALWAYS_INLINE bool 208 malloc_init_a0(void) { 209 if (unlikely(malloc_init_state == malloc_init_uninitialized)) { 210 return malloc_init_hard_a0(); 211 } 212 return false; 213 } 214 215 JEMALLOC_ALWAYS_INLINE bool 216 malloc_init(void) { 217 if (unlikely(!malloc_initialized()) && malloc_init_hard()) { 218 return true; 219 } 220 return false; 221 } 222 223 /* 224 * The a0*() functions are used instead of i{d,}alloc() in situations that 225 * cannot tolerate TLS variable access. 226 */ 227 228 static void * 229 a0ialloc(size_t size, bool zero, bool is_internal) { 230 if (unlikely(malloc_init_a0())) { 231 return NULL; 232 } 233 234 return iallocztm(TSDN_NULL, size, sz_size2index(size), zero, NULL, 235 is_internal, arena_get(TSDN_NULL, 0, true), true); 236 } 237 238 static void 239 a0idalloc(void *ptr, bool is_internal) { 240 idalloctm(TSDN_NULL, ptr, NULL, NULL, is_internal, true); 241 } 242 243 void * 244 a0malloc(size_t size) { 245 return a0ialloc(size, false, true); 246 } 247 248 void 249 a0dalloc(void *ptr) { 250 a0idalloc(ptr, true); 251 } 252 253 /* 254 * FreeBSD's libc uses the bootstrap_*() functions in bootstrap-senstive 255 * situations that cannot tolerate TLS variable access (TLS allocation and very 256 * early internal data structure initialization). 257 */ 258 259 void * 260 bootstrap_malloc(size_t size) { 261 if (unlikely(size == 0)) { 262 size = 1; 263 } 264 265 return a0ialloc(size, false, false); 266 } 267 268 void * 269 bootstrap_calloc(size_t num, size_t size) { 270 size_t num_size; 271 272 num_size = num * size; 273 if (unlikely(num_size == 0)) { 274 assert(num == 0 || size == 0); 275 num_size = 1; 276 } 277 278 return a0ialloc(num_size, true, false); 279 } 280 281 void 282 bootstrap_free(void *ptr) { 283 if (unlikely(ptr == NULL)) { 284 return; 285 } 286 287 a0idalloc(ptr, false); 288 } 289 290 void 291 arena_set(unsigned ind, arena_t *arena) { 292 atomic_store_p(&arenas[ind], arena, ATOMIC_RELEASE); 293 } 294 295 static void 296 narenas_total_set(unsigned narenas) { 297 atomic_store_u(&narenas_total, narenas, ATOMIC_RELEASE); 298 } 299 300 static void 301 narenas_total_inc(void) { 302 atomic_fetch_add_u(&narenas_total, 1, ATOMIC_RELEASE); 303 } 304 305 unsigned 306 narenas_total_get(void) { 307 return atomic_load_u(&narenas_total, ATOMIC_ACQUIRE); 308 } 309 310 /* Create a new arena and insert it into the arenas array at index ind. */ 311 static arena_t * 312 arena_init_locked(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { 313 arena_t *arena; 314 315 assert(ind <= narenas_total_get()); 316 if (ind >= MALLOCX_ARENA_LIMIT) { 317 return NULL; 318 } 319 if (ind == narenas_total_get()) { 320 narenas_total_inc(); 321 } 322 323 /* 324 * Another thread may have already initialized arenas[ind] if it's an 325 * auto arena. 326 */ 327 arena = arena_get(tsdn, ind, false); 328 if (arena != NULL) { 329 assert(ind < narenas_auto); 330 return arena; 331 } 332 333 /* Actually initialize the arena. */ 334 arena = arena_new(tsdn, ind, extent_hooks); 335 336 return arena; 337 } 338 339 static void 340 arena_new_create_background_thread(tsdn_t *tsdn, unsigned ind) { 341 if (ind == 0) { 342 return; 343 } 344 if (have_background_thread) { 345 bool err; 346 malloc_mutex_lock(tsdn, &background_thread_lock); 347 err = background_thread_create(tsdn_tsd(tsdn), ind); 348 malloc_mutex_unlock(tsdn, &background_thread_lock); 349 if (err) { 350 malloc_printf("<jemalloc>: error in background thread " 351 "creation for arena %u. Abort.\n", ind); 352 abort(); 353 } 354 } 355 } 356 357 arena_t * 358 arena_init(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { 359 arena_t *arena; 360 361 malloc_mutex_lock(tsdn, &arenas_lock); 362 arena = arena_init_locked(tsdn, ind, extent_hooks); 363 malloc_mutex_unlock(tsdn, &arenas_lock); 364 365 arena_new_create_background_thread(tsdn, ind); 366 367 return arena; 368 } 369 370 static void 371 arena_bind(tsd_t *tsd, unsigned ind, bool internal) { 372 arena_t *arena = arena_get(tsd_tsdn(tsd), ind, false); 373 arena_nthreads_inc(arena, internal); 374 375 if (internal) { 376 tsd_iarena_set(tsd, arena); 377 } else { 378 tsd_arena_set(tsd, arena); 379 } 380 } 381 382 void 383 arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind) { 384 arena_t *oldarena, *newarena; 385 386 oldarena = arena_get(tsd_tsdn(tsd), oldind, false); 387 newarena = arena_get(tsd_tsdn(tsd), newind, false); 388 arena_nthreads_dec(oldarena, false); 389 arena_nthreads_inc(newarena, false); 390 tsd_arena_set(tsd, newarena); 391 } 392 393 static void 394 arena_unbind(tsd_t *tsd, unsigned ind, bool internal) { 395 arena_t *arena; 396 397 arena = arena_get(tsd_tsdn(tsd), ind, false); 398 arena_nthreads_dec(arena, internal); 399 400 if (internal) { 401 tsd_iarena_set(tsd, NULL); 402 } else { 403 tsd_arena_set(tsd, NULL); 404 } 405 } 406 407 arena_tdata_t * 408 arena_tdata_get_hard(tsd_t *tsd, unsigned ind) { 409 arena_tdata_t *tdata, *arenas_tdata_old; 410 arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd); 411 unsigned narenas_tdata_old, i; 412 unsigned narenas_tdata = tsd_narenas_tdata_get(tsd); 413 unsigned narenas_actual = narenas_total_get(); 414 415 /* 416 * Dissociate old tdata array (and set up for deallocation upon return) 417 * if it's too small. 418 */ 419 if (arenas_tdata != NULL && narenas_tdata < narenas_actual) { 420 arenas_tdata_old = arenas_tdata; 421 narenas_tdata_old = narenas_tdata; 422 arenas_tdata = NULL; 423 narenas_tdata = 0; 424 tsd_arenas_tdata_set(tsd, arenas_tdata); 425 tsd_narenas_tdata_set(tsd, narenas_tdata); 426 } else { 427 arenas_tdata_old = NULL; 428 narenas_tdata_old = 0; 429 } 430 431 /* Allocate tdata array if it's missing. */ 432 if (arenas_tdata == NULL) { 433 bool *arenas_tdata_bypassp = tsd_arenas_tdata_bypassp_get(tsd); 434 narenas_tdata = (ind < narenas_actual) ? narenas_actual : ind+1; 435 436 if (tsd_nominal(tsd) && !*arenas_tdata_bypassp) { 437 *arenas_tdata_bypassp = true; 438 arenas_tdata = (arena_tdata_t *)a0malloc( 439 sizeof(arena_tdata_t) * narenas_tdata); 440 *arenas_tdata_bypassp = false; 441 } 442 if (arenas_tdata == NULL) { 443 tdata = NULL; 444 goto label_return; 445 } 446 assert(tsd_nominal(tsd) && !*arenas_tdata_bypassp); 447 tsd_arenas_tdata_set(tsd, arenas_tdata); 448 tsd_narenas_tdata_set(tsd, narenas_tdata); 449 } 450 451 /* 452 * Copy to tdata array. It's possible that the actual number of arenas 453 * has increased since narenas_total_get() was called above, but that 454 * causes no correctness issues unless two threads concurrently execute 455 * the arenas.create mallctl, which we trust mallctl synchronization to 456 * prevent. 457 */ 458 459 /* Copy/initialize tickers. */ 460 for (i = 0; i < narenas_actual; i++) { 461 if (i < narenas_tdata_old) { 462 ticker_copy(&arenas_tdata[i].decay_ticker, 463 &arenas_tdata_old[i].decay_ticker); 464 } else { 465 ticker_init(&arenas_tdata[i].decay_ticker, 466 DECAY_NTICKS_PER_UPDATE); 467 } 468 } 469 if (narenas_tdata > narenas_actual) { 470 memset(&arenas_tdata[narenas_actual], 0, sizeof(arena_tdata_t) 471 * (narenas_tdata - narenas_actual)); 472 } 473 474 /* Read the refreshed tdata array. */ 475 tdata = &arenas_tdata[ind]; 476 label_return: 477 if (arenas_tdata_old != NULL) { 478 a0dalloc(arenas_tdata_old); 479 } 480 return tdata; 481 } 482 483 /* Slow path, called only by arena_choose(). */ 484 arena_t * 485 arena_choose_hard(tsd_t *tsd, bool internal) { 486 arena_t *ret JEMALLOC_CC_SILENCE_INIT(NULL); 487 488 if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena)) { 489 unsigned choose = percpu_arena_choose(); 490 ret = arena_get(tsd_tsdn(tsd), choose, true); 491 assert(ret != NULL); 492 arena_bind(tsd, arena_ind_get(ret), false); 493 arena_bind(tsd, arena_ind_get(ret), true); 494 495 return ret; 496 } 497 498 if (narenas_auto > 1) { 499 unsigned i, j, choose[2], first_null; 500 bool is_new_arena[2]; 501 502 /* 503 * Determine binding for both non-internal and internal 504 * allocation. 505 * 506 * choose[0]: For application allocation. 507 * choose[1]: For internal metadata allocation. 508 */ 509 510 for (j = 0; j < 2; j++) { 511 choose[j] = 0; 512 is_new_arena[j] = false; 513 } 514 515 first_null = narenas_auto; 516 malloc_mutex_lock(tsd_tsdn(tsd), &arenas_lock); 517 assert(arena_get(tsd_tsdn(tsd), 0, false) != NULL); 518 for (i = 1; i < narenas_auto; i++) { 519 if (arena_get(tsd_tsdn(tsd), i, false) != NULL) { 520 /* 521 * Choose the first arena that has the lowest 522 * number of threads assigned to it. 523 */ 524 for (j = 0; j < 2; j++) { 525 if (arena_nthreads_get(arena_get( 526 tsd_tsdn(tsd), i, false), !!j) < 527 arena_nthreads_get(arena_get( 528 tsd_tsdn(tsd), choose[j], false), 529 !!j)) { 530 choose[j] = i; 531 } 532 } 533 } else if (first_null == narenas_auto) { 534 /* 535 * Record the index of the first uninitialized 536 * arena, in case all extant arenas are in use. 537 * 538 * NB: It is possible for there to be 539 * discontinuities in terms of initialized 540 * versus uninitialized arenas, due to the 541 * "thread.arena" mallctl. 542 */ 543 first_null = i; 544 } 545 } 546 547 for (j = 0; j < 2; j++) { 548 if (arena_nthreads_get(arena_get(tsd_tsdn(tsd), 549 choose[j], false), !!j) == 0 || first_null == 550 narenas_auto) { 551 /* 552 * Use an unloaded arena, or the least loaded 553 * arena if all arenas are already initialized. 554 */ 555 if (!!j == internal) { 556 ret = arena_get(tsd_tsdn(tsd), 557 choose[j], false); 558 } 559 } else { 560 arena_t *arena; 561 562 /* Initialize a new arena. */ 563 choose[j] = first_null; 564 arena = arena_init_locked(tsd_tsdn(tsd), 565 choose[j], 566 (extent_hooks_t *)&extent_hooks_default); 567 if (arena == NULL) { 568 malloc_mutex_unlock(tsd_tsdn(tsd), 569 &arenas_lock); 570 return NULL; 571 } 572 is_new_arena[j] = true; 573 if (!!j == internal) { 574 ret = arena; 575 } 576 } 577 arena_bind(tsd, choose[j], !!j); 578 } 579 malloc_mutex_unlock(tsd_tsdn(tsd), &arenas_lock); 580 581 for (j = 0; j < 2; j++) { 582 if (is_new_arena[j]) { 583 assert(choose[j] > 0); 584 arena_new_create_background_thread( 585 tsd_tsdn(tsd), choose[j]); 586 } 587 } 588 589 } else { 590 ret = arena_get(tsd_tsdn(tsd), 0, false); 591 arena_bind(tsd, 0, false); 592 arena_bind(tsd, 0, true); 593 } 594 595 return ret; 596 } 597 598 void 599 iarena_cleanup(tsd_t *tsd) { 600 arena_t *iarena; 601 602 iarena = tsd_iarena_get(tsd); 603 if (iarena != NULL) { 604 arena_unbind(tsd, arena_ind_get(iarena), true); 605 } 606 } 607 608 void 609 arena_cleanup(tsd_t *tsd) { 610 arena_t *arena; 611 612 arena = tsd_arena_get(tsd); 613 if (arena != NULL) { 614 arena_unbind(tsd, arena_ind_get(arena), false); 615 } 616 } 617 618 void 619 arenas_tdata_cleanup(tsd_t *tsd) { 620 arena_tdata_t *arenas_tdata; 621 622 /* Prevent tsd->arenas_tdata from being (re)created. */ 623 *tsd_arenas_tdata_bypassp_get(tsd) = true; 624 625 arenas_tdata = tsd_arenas_tdata_get(tsd); 626 if (arenas_tdata != NULL) { 627 tsd_arenas_tdata_set(tsd, NULL); 628 a0dalloc(arenas_tdata); 629 } 630 } 631 632 static void 633 stats_print_atexit(void) { 634 if (config_stats) { 635 tsdn_t *tsdn; 636 unsigned narenas, i; 637 638 tsdn = tsdn_fetch(); 639 640 /* 641 * Merge stats from extant threads. This is racy, since 642 * individual threads do not lock when recording tcache stats 643 * events. As a consequence, the final stats may be slightly 644 * out of date by the time they are reported, if other threads 645 * continue to allocate. 646 */ 647 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 648 arena_t *arena = arena_get(tsdn, i, false); 649 if (arena != NULL) { 650 tcache_t *tcache; 651 652 malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); 653 ql_foreach(tcache, &arena->tcache_ql, link) { 654 tcache_stats_merge(tsdn, tcache, arena); 655 } 656 malloc_mutex_unlock(tsdn, 657 &arena->tcache_ql_mtx); 658 } 659 } 660 } 661 je_malloc_stats_print(NULL, NULL, opt_stats_print_opts); 662 } 663 664 /* 665 * Ensure that we don't hold any locks upon entry to or exit from allocator 666 * code (in a "broad" sense that doesn't count a reentrant allocation as an 667 * entrance or exit). 668 */ 669 JEMALLOC_ALWAYS_INLINE void 670 check_entry_exit_locking(tsdn_t *tsdn) { 671 if (!config_debug) { 672 return; 673 } 674 if (tsdn_null(tsdn)) { 675 return; 676 } 677 tsd_t *tsd = tsdn_tsd(tsdn); 678 /* 679 * It's possible we hold locks at entry/exit if we're in a nested 680 * allocation. 681 */ 682 int8_t reentrancy_level = tsd_reentrancy_level_get(tsd); 683 if (reentrancy_level != 0) { 684 return; 685 } 686 witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); 687 } 688 689 /* 690 * End miscellaneous support functions. 691 */ 692 /******************************************************************************/ 693 /* 694 * Begin initialization functions. 695 */ 696 697 static char * 698 jemalloc_secure_getenv(const char *name) { 699 #ifdef JEMALLOC_HAVE_SECURE_GETENV 700 return secure_getenv(name); 701 #else 702 # ifdef JEMALLOC_HAVE_ISSETUGID 703 if (issetugid() != 0) { 704 return NULL; 705 } 706 # endif 707 return getenv(name); 708 #endif 709 } 710 711 static unsigned 712 malloc_ncpus(void) { 713 long result; 714 715 #ifdef _WIN32 716 SYSTEM_INFO si; 717 GetSystemInfo(&si); 718 result = si.dwNumberOfProcessors; 719 #elif defined(JEMALLOC_GLIBC_MALLOC_HOOK) && defined(CPU_COUNT) 720 /* 721 * glibc >= 2.6 has the CPU_COUNT macro. 722 * 723 * glibc's sysconf() uses isspace(). glibc allocates for the first time 724 * *before* setting up the isspace tables. Therefore we need a 725 * different method to get the number of CPUs. 726 */ 727 { 728 cpu_set_t set; 729 730 pthread_getaffinity_np(pthread_self(), sizeof(set), &set); 731 result = CPU_COUNT(&set); 732 } 733 #else 734 result = sysconf(_SC_NPROCESSORS_ONLN); 735 #endif 736 return ((result == -1) ? 1 : (unsigned)result); 737 } 738 739 static void 740 init_opt_stats_print_opts(const char *v, size_t vlen) { 741 size_t opts_len = strlen(opt_stats_print_opts); 742 assert(opts_len <= stats_print_tot_num_options); 743 744 for (size_t i = 0; i < vlen; i++) { 745 switch (v[i]) { 746 #define OPTION(o, v, d, s) case o: break; 747 STATS_PRINT_OPTIONS 748 #undef OPTION 749 default: continue; 750 } 751 752 if (strchr(opt_stats_print_opts, v[i]) != NULL) { 753 /* Ignore repeated. */ 754 continue; 755 } 756 757 opt_stats_print_opts[opts_len++] = v[i]; 758 opt_stats_print_opts[opts_len] = '\0'; 759 assert(opts_len <= stats_print_tot_num_options); 760 } 761 assert(opts_len == strlen(opt_stats_print_opts)); 762 } 763 764 static bool 765 malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, 766 char const **v_p, size_t *vlen_p) { 767 bool accept; 768 const char *opts = *opts_p; 769 770 *k_p = opts; 771 772 for (accept = false; !accept;) { 773 switch (*opts) { 774 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': 775 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': 776 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': 777 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': 778 case 'Y': case 'Z': 779 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': 780 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': 781 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 782 case 's': case 't': case 'u': case 'v': case 'w': case 'x': 783 case 'y': case 'z': 784 case '0': case '1': case '2': case '3': case '4': case '5': 785 case '6': case '7': case '8': case '9': 786 case '_': 787 opts++; 788 break; 789 case ':': 790 opts++; 791 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; 792 *v_p = opts; 793 accept = true; 794 break; 795 case '\0': 796 if (opts != *opts_p) { 797 malloc_write("<jemalloc>: Conf string ends " 798 "with key\n"); 799 } 800 return true; 801 default: 802 malloc_write("<jemalloc>: Malformed conf string\n"); 803 return true; 804 } 805 } 806 807 for (accept = false; !accept;) { 808 switch (*opts) { 809 case ',': 810 opts++; 811 /* 812 * Look ahead one character here, because the next time 813 * this function is called, it will assume that end of 814 * input has been cleanly reached if no input remains, 815 * but we have optimistically already consumed the 816 * comma if one exists. 817 */ 818 if (*opts == '\0') { 819 malloc_write("<jemalloc>: Conf string ends " 820 "with comma\n"); 821 } 822 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; 823 accept = true; 824 break; 825 case '\0': 826 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; 827 accept = true; 828 break; 829 default: 830 opts++; 831 break; 832 } 833 } 834 835 *opts_p = opts; 836 return false; 837 } 838 839 static void 840 malloc_abort_invalid_conf(void) { 841 assert(opt_abort_conf); 842 malloc_printf("<jemalloc>: Abort (abort_conf:true) on invalid conf " 843 "value (see above).\n"); 844 abort(); 845 } 846 847 static void 848 malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, 849 size_t vlen) { 850 malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k, 851 (int)vlen, v); 852 /* If abort_conf is set, error out after processing all options. */ 853 had_conf_error = true; 854 } 855 856 static void 857 malloc_slow_flag_init(void) { 858 /* 859 * Combine the runtime options into malloc_slow for fast path. Called 860 * after processing all the options. 861 */ 862 malloc_slow_flags |= (opt_junk_alloc ? flag_opt_junk_alloc : 0) 863 | (opt_junk_free ? flag_opt_junk_free : 0) 864 | (opt_zero ? flag_opt_zero : 0) 865 | (opt_utrace ? flag_opt_utrace : 0) 866 | (opt_xmalloc ? flag_opt_xmalloc : 0); 867 868 malloc_slow = (malloc_slow_flags != 0); 869 } 870 871 static void 872 malloc_conf_init(void) { 873 unsigned i; 874 char buf[PATH_MAX + 1]; 875 const char *opts, *k, *v; 876 size_t klen, vlen; 877 878 #if defined(__BIONIC__) 879 /* For Android, do not look at files nor environment variables for 880 * config data. 881 */ 882 for (i = 0; i < 2; i++) { 883 #else 884 for (i = 0; i < 4; i++) { 885 #endif 886 /* Get runtime configuration. */ 887 switch (i) { 888 case 0: 889 opts = config_malloc_conf; 890 break; 891 case 1: 892 if (je_malloc_conf != NULL) { 893 /* 894 * Use options that were compiled into the 895 * program. 896 */ 897 opts = je_malloc_conf; 898 } else { 899 /* No configuration specified. */ 900 buf[0] = '\0'; 901 opts = buf; 902 } 903 break; 904 case 2: { 905 ssize_t linklen = 0; 906 #ifndef _WIN32 907 int saved_errno = errno; 908 const char *linkname = 909 # ifdef JEMALLOC_PREFIX 910 "/etc/"JEMALLOC_PREFIX"malloc.conf" 911 # else 912 "/etc/malloc.conf" 913 # endif 914 ; 915 916 /* 917 * Try to use the contents of the "/etc/malloc.conf" 918 * symbolic link's name. 919 */ 920 linklen = readlink(linkname, buf, sizeof(buf) - 1); 921 if (linklen == -1) { 922 /* No configuration specified. */ 923 linklen = 0; 924 /* Restore errno. */ 925 set_errno(saved_errno); 926 } 927 #endif 928 buf[linklen] = '\0'; 929 opts = buf; 930 break; 931 } case 3: { 932 const char *envname = 933 #ifdef JEMALLOC_PREFIX 934 JEMALLOC_CPREFIX"MALLOC_CONF" 935 #else 936 "MALLOC_CONF" 937 #endif 938 ; 939 940 if ((opts = jemalloc_secure_getenv(envname)) != NULL) { 941 /* 942 * Do nothing; opts is already initialized to 943 * the value of the MALLOC_CONF environment 944 * variable. 945 */ 946 } else { 947 /* No configuration specified. */ 948 buf[0] = '\0'; 949 opts = buf; 950 } 951 break; 952 } default: 953 not_reached(); 954 buf[0] = '\0'; 955 opts = buf; 956 } 957 958 while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v, 959 &vlen)) { 960 #define CONF_MATCH(n) \ 961 (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0) 962 #define CONF_MATCH_VALUE(n) \ 963 (sizeof(n)-1 == vlen && strncmp(n, v, vlen) == 0) 964 #define CONF_HANDLE_BOOL(o, n) \ 965 if (CONF_MATCH(n)) { \ 966 if (CONF_MATCH_VALUE("true")) { \ 967 o = true; \ 968 } else if (CONF_MATCH_VALUE("false")) { \ 969 o = false; \ 970 } else { \ 971 malloc_conf_error( \ 972 "Invalid conf value", \ 973 k, klen, v, vlen); \ 974 } \ 975 continue; \ 976 } 977 #define CONF_MIN_no(um, min) false 978 #define CONF_MIN_yes(um, min) ((um) < (min)) 979 #define CONF_MAX_no(um, max) false 980 #define CONF_MAX_yes(um, max) ((um) > (max)) 981 #define CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip) \ 982 if (CONF_MATCH(n)) { \ 983 uintmax_t um; \ 984 char *end; \ 985 \ 986 set_errno(0); \ 987 um = malloc_strtoumax(v, &end, 0); \ 988 if (get_errno() != 0 || (uintptr_t)end -\ 989 (uintptr_t)v != vlen) { \ 990 malloc_conf_error( \ 991 "Invalid conf value", \ 992 k, klen, v, vlen); \ 993 } else if (clip) { \ 994 if (CONF_MIN_##check_min(um, \ 995 (t)(min))) { \ 996 o = (t)(min); \ 997 } else if ( \ 998 CONF_MAX_##check_max(um, \ 999 (t)(max))) { \ 1000 o = (t)(max); \ 1001 } else { \ 1002 o = (t)um; \ 1003 } \ 1004 } else { \ 1005 if (CONF_MIN_##check_min(um, \ 1006 (t)(min)) || \ 1007 CONF_MAX_##check_max(um, \ 1008 (t)(max))) { \ 1009 malloc_conf_error( \ 1010 "Out-of-range " \ 1011 "conf value", \ 1012 k, klen, v, vlen); \ 1013 } else { \ 1014 o = (t)um; \ 1015 } \ 1016 } \ 1017 continue; \ 1018 } 1019 #define CONF_HANDLE_UNSIGNED(o, n, min, max, check_min, check_max, \ 1020 clip) \ 1021 CONF_HANDLE_T_U(unsigned, o, n, min, max, \ 1022 check_min, check_max, clip) 1023 #define CONF_HANDLE_SIZE_T(o, n, min, max, check_min, check_max, clip) \ 1024 CONF_HANDLE_T_U(size_t, o, n, min, max, \ 1025 check_min, check_max, clip) 1026 #define CONF_HANDLE_SSIZE_T(o, n, min, max) \ 1027 if (CONF_MATCH(n)) { \ 1028 long l; \ 1029 char *end; \ 1030 \ 1031 set_errno(0); \ 1032 l = strtol(v, &end, 0); \ 1033 if (get_errno() != 0 || (uintptr_t)end -\ 1034 (uintptr_t)v != vlen) { \ 1035 malloc_conf_error( \ 1036 "Invalid conf value", \ 1037 k, klen, v, vlen); \ 1038 } else if (l < (ssize_t)(min) || l > \ 1039 (ssize_t)(max)) { \ 1040 malloc_conf_error( \ 1041 "Out-of-range conf value", \ 1042 k, klen, v, vlen); \ 1043 } else { \ 1044 o = l; \ 1045 } \ 1046 continue; \ 1047 } 1048 #define CONF_HANDLE_CHAR_P(o, n, d) \ 1049 if (CONF_MATCH(n)) { \ 1050 size_t cpylen = (vlen <= \ 1051 sizeof(o)-1) ? vlen : \ 1052 sizeof(o)-1; \ 1053 strncpy(o, v, cpylen); \ 1054 o[cpylen] = '\0'; \ 1055 continue; \ 1056 } 1057 1058 CONF_HANDLE_BOOL(opt_abort, "abort") 1059 CONF_HANDLE_BOOL(opt_abort_conf, "abort_conf") 1060 if (strncmp("metadata_thp", k, klen) == 0) { 1061 int i; 1062 bool match = false; 1063 for (i = 0; i < metadata_thp_mode_limit; i++) { 1064 if (strncmp(metadata_thp_mode_names[i], 1065 v, vlen) == 0) { 1066 opt_metadata_thp = i; 1067 match = true; 1068 break; 1069 } 1070 } 1071 if (!match) { 1072 malloc_conf_error("Invalid conf value", 1073 k, klen, v, vlen); 1074 } 1075 continue; 1076 } 1077 CONF_HANDLE_BOOL(opt_retain, "retain") 1078 if (strncmp("dss", k, klen) == 0) { 1079 int i; 1080 bool match = false; 1081 for (i = 0; i < dss_prec_limit; i++) { 1082 if (strncmp(dss_prec_names[i], v, vlen) 1083 == 0) { 1084 if (extent_dss_prec_set(i)) { 1085 malloc_conf_error( 1086 "Error setting dss", 1087 k, klen, v, vlen); 1088 } else { 1089 opt_dss = 1090 dss_prec_names[i]; 1091 match = true; 1092 break; 1093 } 1094 } 1095 } 1096 if (!match) { 1097 malloc_conf_error("Invalid conf value", 1098 k, klen, v, vlen); 1099 } 1100 continue; 1101 } 1102 CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1, 1103 UINT_MAX, yes, no, false) 1104 CONF_HANDLE_SSIZE_T(opt_dirty_decay_ms, 1105 "dirty_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) < 1106 QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) : 1107 SSIZE_MAX); 1108 CONF_HANDLE_SSIZE_T(opt_muzzy_decay_ms, 1109 "muzzy_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) < 1110 QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) : 1111 SSIZE_MAX); 1112 CONF_HANDLE_BOOL(opt_stats_print, "stats_print") 1113 if (CONF_MATCH("stats_print_opts")) { 1114 init_opt_stats_print_opts(v, vlen); 1115 continue; 1116 } 1117 if (config_fill) { 1118 if (CONF_MATCH("junk")) { 1119 if (CONF_MATCH_VALUE("true")) { 1120 opt_junk = "true"; 1121 opt_junk_alloc = opt_junk_free = 1122 true; 1123 } else if (CONF_MATCH_VALUE("false")) { 1124 opt_junk = "false"; 1125 opt_junk_alloc = opt_junk_free = 1126 false; 1127 } else if (CONF_MATCH_VALUE("alloc")) { 1128 opt_junk = "alloc"; 1129 opt_junk_alloc = true; 1130 opt_junk_free = false; 1131 } else if (CONF_MATCH_VALUE("free")) { 1132 opt_junk = "free"; 1133 opt_junk_alloc = false; 1134 opt_junk_free = true; 1135 } else { 1136 malloc_conf_error( 1137 "Invalid conf value", k, 1138 klen, v, vlen); 1139 } 1140 continue; 1141 } 1142 CONF_HANDLE_BOOL(opt_zero, "zero") 1143 } 1144 if (config_utrace) { 1145 CONF_HANDLE_BOOL(opt_utrace, "utrace") 1146 } 1147 if (config_xmalloc) { 1148 CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc") 1149 } 1150 CONF_HANDLE_BOOL(opt_tcache, "tcache") 1151 CONF_HANDLE_SIZE_T(opt_lg_extent_max_active_fit, 1152 "lg_extent_max_active_fit", 0, 1153 (sizeof(size_t) << 3), yes, yes, false) 1154 CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, "lg_tcache_max", 1155 -1, (sizeof(size_t) << 3) - 1) 1156 if (strncmp("percpu_arena", k, klen) == 0) { 1157 bool match = false; 1158 for (int i = percpu_arena_mode_names_base; i < 1159 percpu_arena_mode_names_limit; i++) { 1160 if (strncmp(percpu_arena_mode_names[i], 1161 v, vlen) == 0) { 1162 if (!have_percpu_arena) { 1163 malloc_conf_error( 1164 "No getcpu support", 1165 k, klen, v, vlen); 1166 } 1167 opt_percpu_arena = i; 1168 match = true; 1169 break; 1170 } 1171 } 1172 if (!match) { 1173 malloc_conf_error("Invalid conf value", 1174 k, klen, v, vlen); 1175 } 1176 continue; 1177 } 1178 CONF_HANDLE_BOOL(opt_background_thread, 1179 "background_thread"); 1180 CONF_HANDLE_SIZE_T(opt_max_background_threads, 1181 "max_background_threads", 1, 1182 opt_max_background_threads, yes, yes, 1183 true); 1184 if (config_prof) { 1185 CONF_HANDLE_BOOL(opt_prof, "prof") 1186 CONF_HANDLE_CHAR_P(opt_prof_prefix, 1187 "prof_prefix", "jeprof") 1188 CONF_HANDLE_BOOL(opt_prof_active, "prof_active") 1189 CONF_HANDLE_BOOL(opt_prof_thread_active_init, 1190 "prof_thread_active_init") 1191 CONF_HANDLE_SIZE_T(opt_lg_prof_sample, 1192 "lg_prof_sample", 0, (sizeof(uint64_t) << 3) 1193 - 1, no, yes, true) 1194 CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum") 1195 CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, 1196 "lg_prof_interval", -1, 1197 (sizeof(uint64_t) << 3) - 1) 1198 CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump") 1199 CONF_HANDLE_BOOL(opt_prof_final, "prof_final") 1200 CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak") 1201 } 1202 if (config_log) { 1203 if (CONF_MATCH("log")) { 1204 size_t cpylen = ( 1205 vlen <= sizeof(log_var_names) ? 1206 vlen : sizeof(log_var_names) - 1); 1207 strncpy(log_var_names, v, cpylen); 1208 log_var_names[cpylen] = '\0'; 1209 continue; 1210 } 1211 } 1212 if (CONF_MATCH("thp")) { 1213 bool match = false; 1214 for (int i = 0; i < thp_mode_names_limit; i++) { 1215 if (strncmp(thp_mode_names[i],v, vlen) 1216 == 0) { 1217 if (!have_madvise_huge) { 1218 malloc_conf_error( 1219 "No THP support", 1220 k, klen, v, vlen); 1221 } 1222 opt_thp = i; 1223 match = true; 1224 break; 1225 } 1226 } 1227 if (!match) { 1228 malloc_conf_error("Invalid conf value", 1229 k, klen, v, vlen); 1230 } 1231 continue; 1232 } 1233 malloc_conf_error("Invalid conf pair", k, klen, v, 1234 vlen); 1235 #undef CONF_MATCH 1236 #undef CONF_MATCH_VALUE 1237 #undef CONF_HANDLE_BOOL 1238 #undef CONF_MIN_no 1239 #undef CONF_MIN_yes 1240 #undef CONF_MAX_no 1241 #undef CONF_MAX_yes 1242 #undef CONF_HANDLE_T_U 1243 #undef CONF_HANDLE_UNSIGNED 1244 #undef CONF_HANDLE_SIZE_T 1245 #undef CONF_HANDLE_SSIZE_T 1246 #undef CONF_HANDLE_CHAR_P 1247 } 1248 if (opt_abort_conf && had_conf_error) { 1249 malloc_abort_invalid_conf(); 1250 } 1251 } 1252 atomic_store_b(&log_init_done, true, ATOMIC_RELEASE); 1253 } 1254 1255 static bool 1256 malloc_init_hard_needed(void) { 1257 if (malloc_initialized() || (IS_INITIALIZER && malloc_init_state == 1258 malloc_init_recursible)) { 1259 /* 1260 * Another thread initialized the allocator before this one 1261 * acquired init_lock, or this thread is the initializing 1262 * thread, and it is recursively allocating. 1263 */ 1264 return false; 1265 } 1266 #ifdef JEMALLOC_THREADED_INIT 1267 if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) { 1268 /* Busy-wait until the initializing thread completes. */ 1269 spin_t spinner = SPIN_INITIALIZER; 1270 do { 1271 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1272 spin_adaptive(&spinner); 1273 malloc_mutex_lock(TSDN_NULL, &init_lock); 1274 } while (!malloc_initialized()); 1275 return false; 1276 } 1277 #endif 1278 return true; 1279 } 1280 1281 static bool 1282 malloc_init_hard_a0_locked() { 1283 malloc_initializer = INITIALIZER; 1284 1285 if (config_prof) { 1286 prof_boot0(); 1287 } 1288 malloc_conf_init(); 1289 if (opt_stats_print) { 1290 /* Print statistics at exit. */ 1291 if (atexit(stats_print_atexit) != 0) { 1292 malloc_write("<jemalloc>: Error in atexit()\n"); 1293 if (opt_abort) { 1294 abort(); 1295 } 1296 } 1297 } 1298 if (pages_boot()) { 1299 return true; 1300 } 1301 if (base_boot(TSDN_NULL)) { 1302 return true; 1303 } 1304 if (extent_boot()) { 1305 return true; 1306 } 1307 if (ctl_boot()) { 1308 return true; 1309 } 1310 if (config_prof) { 1311 prof_boot1(); 1312 } 1313 arena_boot(); 1314 if (tcache_boot(TSDN_NULL)) { 1315 return true; 1316 } 1317 if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS, 1318 malloc_mutex_rank_exclusive)) { 1319 return true; 1320 } 1321 /* 1322 * Create enough scaffolding to allow recursive allocation in 1323 * malloc_ncpus(). 1324 */ 1325 narenas_auto = 1; 1326 memset(arenas, 0, sizeof(arena_t *) * narenas_auto); 1327 /* 1328 * Initialize one arena here. The rest are lazily created in 1329 * arena_choose_hard(). 1330 */ 1331 if (arena_init(TSDN_NULL, 0, (extent_hooks_t *)&extent_hooks_default) 1332 == NULL) { 1333 return true; 1334 } 1335 a0 = arena_get(TSDN_NULL, 0, false); 1336 malloc_init_state = malloc_init_a0_initialized; 1337 1338 return false; 1339 } 1340 1341 static bool 1342 malloc_init_hard_a0(void) { 1343 bool ret; 1344 1345 malloc_mutex_lock(TSDN_NULL, &init_lock); 1346 ret = malloc_init_hard_a0_locked(); 1347 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1348 return ret; 1349 } 1350 1351 /* Initialize data structures which may trigger recursive allocation. */ 1352 static bool 1353 malloc_init_hard_recursible(void) { 1354 malloc_init_state = malloc_init_recursible; 1355 1356 #if defined(__BIONIC__) && defined(ANDROID_NUM_ARENAS) 1357 /* Hardcode since this value won't be used. */ 1358 ncpus = 2; 1359 #else 1360 ncpus = malloc_ncpus(); 1361 #endif 1362 1363 #if (defined(JEMALLOC_HAVE_PTHREAD_ATFORK) && !defined(JEMALLOC_MUTEX_INIT_CB) \ 1364 && !defined(JEMALLOC_ZONE) && !defined(_WIN32) && \ 1365 !defined(__native_client__)) 1366 /* LinuxThreads' pthread_atfork() allocates. */ 1367 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, 1368 jemalloc_postfork_child) != 0) { 1369 malloc_write("<jemalloc>: Error in pthread_atfork()\n"); 1370 if (opt_abort) { 1371 abort(); 1372 } 1373 return true; 1374 } 1375 #endif 1376 1377 if (background_thread_boot0()) { 1378 return true; 1379 } 1380 1381 return false; 1382 } 1383 1384 static unsigned 1385 malloc_narenas_default(void) { 1386 #if defined(ANDROID_NUM_ARENAS) 1387 return ANDROID_NUM_ARENAS; 1388 #else 1389 assert(ncpus > 0); 1390 /* 1391 * For SMP systems, create more than one arena per CPU by 1392 * default. 1393 */ 1394 if (ncpus > 1) { 1395 return ncpus << 2; 1396 } else { 1397 return 1; 1398 } 1399 #endif 1400 } 1401 1402 static percpu_arena_mode_t 1403 percpu_arena_as_initialized(percpu_arena_mode_t mode) { 1404 assert(!malloc_initialized()); 1405 assert(mode <= percpu_arena_disabled); 1406 1407 if (mode != percpu_arena_disabled) { 1408 mode += percpu_arena_mode_enabled_base; 1409 } 1410 1411 return mode; 1412 } 1413 1414 static bool 1415 malloc_init_narenas(void) { 1416 assert(ncpus > 0); 1417 1418 if (opt_percpu_arena != percpu_arena_disabled) { 1419 if (!have_percpu_arena || malloc_getcpu() < 0) { 1420 opt_percpu_arena = percpu_arena_disabled; 1421 malloc_printf("<jemalloc>: perCPU arena getcpu() not " 1422 "available. Setting narenas to %u.\n", opt_narenas ? 1423 opt_narenas : malloc_narenas_default()); 1424 if (opt_abort) { 1425 abort(); 1426 } 1427 } else { 1428 if (ncpus >= MALLOCX_ARENA_LIMIT) { 1429 malloc_printf("<jemalloc>: narenas w/ percpu" 1430 "arena beyond limit (%d)\n", ncpus); 1431 if (opt_abort) { 1432 abort(); 1433 } 1434 return true; 1435 } 1436 /* NB: opt_percpu_arena isn't fully initialized yet. */ 1437 if (percpu_arena_as_initialized(opt_percpu_arena) == 1438 per_phycpu_arena && ncpus % 2 != 0) { 1439 malloc_printf("<jemalloc>: invalid " 1440 "configuration -- per physical CPU arena " 1441 "with odd number (%u) of CPUs (no hyper " 1442 "threading?).\n", ncpus); 1443 if (opt_abort) 1444 abort(); 1445 } 1446 unsigned n = percpu_arena_ind_limit( 1447 percpu_arena_as_initialized(opt_percpu_arena)); 1448 if (opt_narenas < n) { 1449 /* 1450 * If narenas is specified with percpu_arena 1451 * enabled, actual narenas is set as the greater 1452 * of the two. percpu_arena_choose will be free 1453 * to use any of the arenas based on CPU 1454 * id. This is conservative (at a small cost) 1455 * but ensures correctness. 1456 * 1457 * If for some reason the ncpus determined at 1458 * boot is not the actual number (e.g. because 1459 * of affinity setting from numactl), reserving 1460 * narenas this way provides a workaround for 1461 * percpu_arena. 1462 */ 1463 opt_narenas = n; 1464 } 1465 } 1466 } 1467 if (opt_narenas == 0) { 1468 opt_narenas = malloc_narenas_default(); 1469 } 1470 assert(opt_narenas > 0); 1471 1472 narenas_auto = opt_narenas; 1473 /* 1474 * Limit the number of arenas to the indexing range of MALLOCX_ARENA(). 1475 */ 1476 if (narenas_auto >= MALLOCX_ARENA_LIMIT) { 1477 narenas_auto = MALLOCX_ARENA_LIMIT - 1; 1478 malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n", 1479 narenas_auto); 1480 } 1481 narenas_total_set(narenas_auto); 1482 1483 return false; 1484 } 1485 1486 static void 1487 malloc_init_percpu(void) { 1488 opt_percpu_arena = percpu_arena_as_initialized(opt_percpu_arena); 1489 } 1490 1491 static bool 1492 malloc_init_hard_finish(void) { 1493 if (malloc_mutex_boot()) { 1494 return true; 1495 } 1496 1497 malloc_init_state = malloc_init_initialized; 1498 malloc_slow_flag_init(); 1499 1500 return false; 1501 } 1502 1503 static void 1504 malloc_init_hard_cleanup(tsdn_t *tsdn, bool reentrancy_set) { 1505 malloc_mutex_assert_owner(tsdn, &init_lock); 1506 malloc_mutex_unlock(tsdn, &init_lock); 1507 if (reentrancy_set) { 1508 assert(!tsdn_null(tsdn)); 1509 tsd_t *tsd = tsdn_tsd(tsdn); 1510 assert(tsd_reentrancy_level_get(tsd) > 0); 1511 post_reentrancy(tsd); 1512 } 1513 } 1514 1515 static bool 1516 malloc_init_hard(void) { 1517 tsd_t *tsd; 1518 1519 #if defined(_WIN32) && _WIN32_WINNT < 0x0600 1520 _init_init_lock(); 1521 #endif 1522 malloc_mutex_lock(TSDN_NULL, &init_lock); 1523 1524 #define UNLOCK_RETURN(tsdn, ret, reentrancy) \ 1525 malloc_init_hard_cleanup(tsdn, reentrancy); \ 1526 return ret; 1527 1528 if (!malloc_init_hard_needed()) { 1529 UNLOCK_RETURN(TSDN_NULL, false, false) 1530 } 1531 1532 if (malloc_init_state != malloc_init_a0_initialized && 1533 malloc_init_hard_a0_locked()) { 1534 UNLOCK_RETURN(TSDN_NULL, true, false) 1535 } 1536 1537 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1538 /* Recursive allocation relies on functional tsd. */ 1539 tsd = malloc_tsd_boot0(); 1540 if (tsd == NULL) { 1541 return true; 1542 } 1543 if (malloc_init_hard_recursible()) { 1544 return true; 1545 } 1546 1547 malloc_mutex_lock(tsd_tsdn(tsd), &init_lock); 1548 /* Set reentrancy level to 1 during init. */ 1549 pre_reentrancy(tsd, NULL); 1550 /* Initialize narenas before prof_boot2 (for allocation). */ 1551 if (malloc_init_narenas() || background_thread_boot1(tsd_tsdn(tsd))) { 1552 UNLOCK_RETURN(tsd_tsdn(tsd), true, true) 1553 } 1554 if (config_prof && prof_boot2(tsd)) { 1555 UNLOCK_RETURN(tsd_tsdn(tsd), true, true) 1556 } 1557 1558 malloc_init_percpu(); 1559 1560 if (malloc_init_hard_finish()) { 1561 UNLOCK_RETURN(tsd_tsdn(tsd), true, true) 1562 } 1563 post_reentrancy(tsd); 1564 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock); 1565 1566 witness_assert_lockless(witness_tsd_tsdn( 1567 tsd_witness_tsdp_get_unsafe(tsd))); 1568 malloc_tsd_boot1(); 1569 /* Update TSD after tsd_boot1. */ 1570 tsd = tsd_fetch(); 1571 if (opt_background_thread) { 1572 assert(have_background_thread); 1573 /* 1574 * Need to finish init & unlock first before creating background 1575 * threads (pthread_create depends on malloc). ctl_init (which 1576 * sets isthreaded) needs to be called without holding any lock. 1577 */ 1578 background_thread_ctl_init(tsd_tsdn(tsd)); 1579 1580 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); 1581 bool err = background_thread_create(tsd, 0); 1582 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); 1583 if (err) { 1584 return true; 1585 } 1586 } 1587 #undef UNLOCK_RETURN 1588 return false; 1589 } 1590 1591 /* 1592 * End initialization functions. 1593 */ 1594 /******************************************************************************/ 1595 /* 1596 * Begin allocation-path internal functions and data structures. 1597 */ 1598 1599 /* 1600 * Settings determined by the documented behavior of the allocation functions. 1601 */ 1602 typedef struct static_opts_s static_opts_t; 1603 struct static_opts_s { 1604 /* Whether or not allocation size may overflow. */ 1605 bool may_overflow; 1606 /* Whether or not allocations of size 0 should be treated as size 1. */ 1607 bool bump_empty_alloc; 1608 /* 1609 * Whether to assert that allocations are not of size 0 (after any 1610 * bumping). 1611 */ 1612 bool assert_nonempty_alloc; 1613 1614 /* 1615 * Whether or not to modify the 'result' argument to malloc in case of 1616 * error. 1617 */ 1618 bool null_out_result_on_error; 1619 /* Whether to set errno when we encounter an error condition. */ 1620 bool set_errno_on_error; 1621 1622 /* 1623 * The minimum valid alignment for functions requesting aligned storage. 1624 */ 1625 size_t min_alignment; 1626 1627 /* The error string to use if we oom. */ 1628 const char *oom_string; 1629 /* The error string to use if the passed-in alignment is invalid. */ 1630 const char *invalid_alignment_string; 1631 1632 /* 1633 * False if we're configured to skip some time-consuming operations. 1634 * 1635 * This isn't really a malloc "behavior", but it acts as a useful 1636 * summary of several other static (or at least, static after program 1637 * initialization) options. 1638 */ 1639 bool slow; 1640 }; 1641 1642 JEMALLOC_ALWAYS_INLINE void 1643 static_opts_init(static_opts_t *static_opts) { 1644 static_opts->may_overflow = false; 1645 static_opts->bump_empty_alloc = false; 1646 static_opts->assert_nonempty_alloc = false; 1647 static_opts->null_out_result_on_error = false; 1648 static_opts->set_errno_on_error = false; 1649 static_opts->min_alignment = 0; 1650 static_opts->oom_string = ""; 1651 static_opts->invalid_alignment_string = ""; 1652 static_opts->slow = false; 1653 } 1654 1655 /* 1656 * These correspond to the macros in jemalloc/jemalloc_macros.h. Broadly, we 1657 * should have one constant here per magic value there. Note however that the 1658 * representations need not be related. 1659 */ 1660 #define TCACHE_IND_NONE ((unsigned)-1) 1661 #define TCACHE_IND_AUTOMATIC ((unsigned)-2) 1662 #define ARENA_IND_AUTOMATIC ((unsigned)-1) 1663 1664 typedef struct dynamic_opts_s dynamic_opts_t; 1665 struct dynamic_opts_s { 1666 void **result; 1667 size_t num_items; 1668 size_t item_size; 1669 size_t alignment; 1670 bool zero; 1671 unsigned tcache_ind; 1672 unsigned arena_ind; 1673 }; 1674 1675 JEMALLOC_ALWAYS_INLINE void 1676 dynamic_opts_init(dynamic_opts_t *dynamic_opts) { 1677 dynamic_opts->result = NULL; 1678 dynamic_opts->num_items = 0; 1679 dynamic_opts->item_size = 0; 1680 dynamic_opts->alignment = 0; 1681 dynamic_opts->zero = false; 1682 dynamic_opts->tcache_ind = TCACHE_IND_AUTOMATIC; 1683 dynamic_opts->arena_ind = ARENA_IND_AUTOMATIC; 1684 } 1685 1686 /* ind is ignored if dopts->alignment > 0. */ 1687 JEMALLOC_ALWAYS_INLINE void * 1688 imalloc_no_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd, 1689 size_t size, size_t usize, szind_t ind) { 1690 tcache_t *tcache; 1691 arena_t *arena; 1692 1693 /* Fill in the tcache. */ 1694 if (dopts->tcache_ind == TCACHE_IND_AUTOMATIC) { 1695 if (likely(!sopts->slow)) { 1696 /* Getting tcache ptr unconditionally. */ 1697 tcache = tsd_tcachep_get(tsd); 1698 assert(tcache == tcache_get(tsd)); 1699 } else { 1700 tcache = tcache_get(tsd); 1701 } 1702 } else if (dopts->tcache_ind == TCACHE_IND_NONE) { 1703 tcache = NULL; 1704 } else { 1705 tcache = tcaches_get(tsd, dopts->tcache_ind); 1706 } 1707 1708 /* Fill in the arena. */ 1709 if (dopts->arena_ind == ARENA_IND_AUTOMATIC) { 1710 /* 1711 * In case of automatic arena management, we defer arena 1712 * computation until as late as we can, hoping to fill the 1713 * allocation out of the tcache. 1714 */ 1715 arena = NULL; 1716 } else { 1717 arena = arena_get(tsd_tsdn(tsd), dopts->arena_ind, true); 1718 } 1719 1720 if (unlikely(dopts->alignment != 0)) { 1721 return ipalloct(tsd_tsdn(tsd), usize, dopts->alignment, 1722 dopts->zero, tcache, arena); 1723 } 1724 1725 return iallocztm(tsd_tsdn(tsd), size, ind, dopts->zero, tcache, false, 1726 arena, sopts->slow); 1727 } 1728 1729 JEMALLOC_ALWAYS_INLINE void * 1730 imalloc_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd, 1731 size_t usize, szind_t ind) { 1732 void *ret; 1733 1734 /* 1735 * For small allocations, sampling bumps the usize. If so, we allocate 1736 * from the ind_large bucket. 1737 */ 1738 szind_t ind_large; 1739 size_t bumped_usize = usize; 1740 1741 if (usize <= SMALL_MAXCLASS) { 1742 assert(((dopts->alignment == 0) ? sz_s2u(LARGE_MINCLASS) : 1743 sz_sa2u(LARGE_MINCLASS, dopts->alignment)) 1744 == LARGE_MINCLASS); 1745 ind_large = sz_size2index(LARGE_MINCLASS); 1746 bumped_usize = sz_s2u(LARGE_MINCLASS); 1747 ret = imalloc_no_sample(sopts, dopts, tsd, bumped_usize, 1748 bumped_usize, ind_large); 1749 if (unlikely(ret == NULL)) { 1750 return NULL; 1751 } 1752 arena_prof_promote(tsd_tsdn(tsd), ret, usize); 1753 } else { 1754 ret = imalloc_no_sample(sopts, dopts, tsd, usize, usize, ind); 1755 } 1756 1757 return ret; 1758 } 1759 1760 /* 1761 * Returns true if the allocation will overflow, and false otherwise. Sets 1762 * *size to the product either way. 1763 */ 1764 JEMALLOC_ALWAYS_INLINE bool 1765 compute_size_with_overflow(bool may_overflow, dynamic_opts_t *dopts, 1766 size_t *size) { 1767 /* 1768 * This function is just num_items * item_size, except that we may have 1769 * to check for overflow. 1770 */ 1771 1772 if (!may_overflow) { 1773 assert(dopts->num_items == 1); 1774 *size = dopts->item_size; 1775 return false; 1776 } 1777 1778 /* A size_t with its high-half bits all set to 1. */ 1779 static const size_t high_bits = SIZE_T_MAX << (sizeof(size_t) * 8 / 2); 1780 1781 *size = dopts->item_size * dopts->num_items; 1782 1783 if (unlikely(*size == 0)) { 1784 return (dopts->num_items != 0 && dopts->item_size != 0); 1785 } 1786 1787 /* 1788 * We got a non-zero size, but we don't know if we overflowed to get 1789 * there. To avoid having to do a divide, we'll be clever and note that 1790 * if both A and B can be represented in N/2 bits, then their product 1791 * can be represented in N bits (without the possibility of overflow). 1792 */ 1793 if (likely((high_bits & (dopts->num_items | dopts->item_size)) == 0)) { 1794 return false; 1795 } 1796 if (likely(*size / dopts->item_size == dopts->num_items)) { 1797 return false; 1798 } 1799 return true; 1800 } 1801 1802 JEMALLOC_ALWAYS_INLINE int 1803 imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { 1804 /* Where the actual allocated memory will live. */ 1805 void *allocation = NULL; 1806 /* Filled in by compute_size_with_overflow below. */ 1807 size_t size = 0; 1808 /* 1809 * For unaligned allocations, we need only ind. For aligned 1810 * allocations, or in case of stats or profiling we need usize. 1811 * 1812 * These are actually dead stores, in that their values are reset before 1813 * any branch on their value is taken. Sometimes though, it's 1814 * convenient to pass them as arguments before this point. To avoid 1815 * undefined behavior then, we initialize them with dummy stores. 1816 */ 1817 szind_t ind = 0; 1818 size_t usize = 0; 1819 1820 /* Reentrancy is only checked on slow path. */ 1821 int8_t reentrancy_level; 1822 1823 /* Compute the amount of memory the user wants. */ 1824 if (unlikely(compute_size_with_overflow(sopts->may_overflow, dopts, 1825 &size))) { 1826 goto label_oom; 1827 } 1828 1829 /* Validate the user input. */ 1830 if (sopts->bump_empty_alloc) { 1831 if (unlikely(size == 0)) { 1832 size = 1; 1833 } 1834 } 1835 1836 if (sopts->assert_nonempty_alloc) { 1837 assert (size != 0); 1838 } 1839 1840 if (unlikely(dopts->alignment < sopts->min_alignment 1841 || (dopts->alignment & (dopts->alignment - 1)) != 0)) { 1842 goto label_invalid_alignment; 1843 } 1844 1845 /* This is the beginning of the "core" algorithm. */ 1846 1847 if (dopts->alignment == 0) { 1848 ind = sz_size2index(size); 1849 if (unlikely(ind >= NSIZES)) { 1850 goto label_oom; 1851 } 1852 if (config_stats || (config_prof && opt_prof)) { 1853 usize = sz_index2size(ind); 1854 assert(usize > 0 && usize <= LARGE_MAXCLASS); 1855 } 1856 } else { 1857 usize = sz_sa2u(size, dopts->alignment); 1858 if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { 1859 goto label_oom; 1860 } 1861 } 1862 1863 check_entry_exit_locking(tsd_tsdn(tsd)); 1864 1865 /* 1866 * If we need to handle reentrancy, we can do it out of a 1867 * known-initialized arena (i.e. arena 0). 1868 */ 1869 reentrancy_level = tsd_reentrancy_level_get(tsd); 1870 if (sopts->slow && unlikely(reentrancy_level > 0)) { 1871 /* 1872 * We should never specify particular arenas or tcaches from 1873 * within our internal allocations. 1874 */ 1875 assert(dopts->tcache_ind == TCACHE_IND_AUTOMATIC || 1876 dopts->tcache_ind == TCACHE_IND_NONE); 1877 assert(dopts->arena_ind == ARENA_IND_AUTOMATIC); 1878 dopts->tcache_ind = TCACHE_IND_NONE; 1879 /* We know that arena 0 has already been initialized. */ 1880 dopts->arena_ind = 0; 1881 } 1882 1883 /* If profiling is on, get our profiling context. */ 1884 if (config_prof && opt_prof) { 1885 /* 1886 * Note that if we're going down this path, usize must have been 1887 * initialized in the previous if statement. 1888 */ 1889 prof_tctx_t *tctx = prof_alloc_prep( 1890 tsd, usize, prof_active_get_unlocked(), true); 1891 1892 alloc_ctx_t alloc_ctx; 1893 if (likely((uintptr_t)tctx == (uintptr_t)1U)) { 1894 alloc_ctx.slab = (usize <= SMALL_MAXCLASS); 1895 allocation = imalloc_no_sample( 1896 sopts, dopts, tsd, usize, usize, ind); 1897 } else if ((uintptr_t)tctx > (uintptr_t)1U) { 1898 /* 1899 * Note that ind might still be 0 here. This is fine; 1900 * imalloc_sample ignores ind if dopts->alignment > 0. 1901 */ 1902 allocation = imalloc_sample( 1903 sopts, dopts, tsd, usize, ind); 1904 alloc_ctx.slab = false; 1905 } else { 1906 allocation = NULL; 1907 } 1908 1909 if (unlikely(allocation == NULL)) { 1910 prof_alloc_rollback(tsd, tctx, true); 1911 goto label_oom; 1912 } 1913 prof_malloc(tsd_tsdn(tsd), allocation, usize, &alloc_ctx, tctx); 1914 } else { 1915 /* 1916 * If dopts->alignment > 0, then ind is still 0, but usize was 1917 * computed in the previous if statement. Down the positive 1918 * alignment path, imalloc_no_sample ignores ind and size 1919 * (relying only on usize). 1920 */ 1921 allocation = imalloc_no_sample(sopts, dopts, tsd, size, usize, 1922 ind); 1923 if (unlikely(allocation == NULL)) { 1924 goto label_oom; 1925 } 1926 } 1927 1928 /* 1929 * Allocation has been done at this point. We still have some 1930 * post-allocation work to do though. 1931 */ 1932 assert(dopts->alignment == 0 1933 || ((uintptr_t)allocation & (dopts->alignment - 1)) == ZU(0)); 1934 1935 if (config_stats) { 1936 assert(usize == isalloc(tsd_tsdn(tsd), allocation)); 1937 *tsd_thread_allocatedp_get(tsd) += usize; 1938 } 1939 1940 if (sopts->slow) { 1941 UTRACE(0, size, allocation); 1942 } 1943 1944 /* Success! */ 1945 check_entry_exit_locking(tsd_tsdn(tsd)); 1946 *dopts->result = allocation; 1947 return 0; 1948 1949 label_oom: 1950 if (unlikely(sopts->slow) && config_xmalloc && unlikely(opt_xmalloc)) { 1951 malloc_write(sopts->oom_string); 1952 abort(); 1953 } 1954 1955 if (sopts->slow) { 1956 UTRACE(NULL, size, NULL); 1957 } 1958 1959 check_entry_exit_locking(tsd_tsdn(tsd)); 1960 1961 if (sopts->set_errno_on_error) { 1962 set_errno(ENOMEM); 1963 } 1964 1965 if (sopts->null_out_result_on_error) { 1966 *dopts->result = NULL; 1967 } 1968 1969 return ENOMEM; 1970 1971 /* 1972 * This label is only jumped to by one goto; we move it out of line 1973 * anyways to avoid obscuring the non-error paths, and for symmetry with 1974 * the oom case. 1975 */ 1976 label_invalid_alignment: 1977 if (config_xmalloc && unlikely(opt_xmalloc)) { 1978 malloc_write(sopts->invalid_alignment_string); 1979 abort(); 1980 } 1981 1982 if (sopts->set_errno_on_error) { 1983 set_errno(EINVAL); 1984 } 1985 1986 if (sopts->slow) { 1987 UTRACE(NULL, size, NULL); 1988 } 1989 1990 check_entry_exit_locking(tsd_tsdn(tsd)); 1991 1992 if (sopts->null_out_result_on_error) { 1993 *dopts->result = NULL; 1994 } 1995 1996 return EINVAL; 1997 } 1998 1999 /* Returns the errno-style error code of the allocation. */ 2000 JEMALLOC_ALWAYS_INLINE int 2001 imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) { 2002 if (unlikely(!malloc_initialized()) && unlikely(malloc_init())) { 2003 if (config_xmalloc && unlikely(opt_xmalloc)) { 2004 malloc_write(sopts->oom_string); 2005 abort(); 2006 } 2007 UTRACE(NULL, dopts->num_items * dopts->item_size, NULL); 2008 set_errno(ENOMEM); 2009 *dopts->result = NULL; 2010 2011 return ENOMEM; 2012 } 2013 2014 /* We always need the tsd. Let's grab it right away. */ 2015 tsd_t *tsd = tsd_fetch(); 2016 assert(tsd); 2017 if (likely(tsd_fast(tsd))) { 2018 /* Fast and common path. */ 2019 tsd_assert_fast(tsd); 2020 sopts->slow = false; 2021 return imalloc_body(sopts, dopts, tsd); 2022 } else { 2023 sopts->slow = true; 2024 return imalloc_body(sopts, dopts, tsd); 2025 } 2026 } 2027 /******************************************************************************/ 2028 /* 2029 * Begin malloc(3)-compatible functions. 2030 */ 2031 2032 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2033 void JEMALLOC_NOTHROW * 2034 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) 2035 je_malloc(size_t size) { 2036 void *ret; 2037 static_opts_t sopts; 2038 dynamic_opts_t dopts; 2039 2040 LOG("core.malloc.entry", "size: %zu", size); 2041 2042 static_opts_init(&sopts); 2043 dynamic_opts_init(&dopts); 2044 2045 sopts.bump_empty_alloc = true; 2046 sopts.null_out_result_on_error = true; 2047 sopts.set_errno_on_error = true; 2048 sopts.oom_string = "<jemalloc>: Error in malloc(): out of memory\n"; 2049 2050 dopts.result = &ret; 2051 dopts.num_items = 1; 2052 dopts.item_size = size; 2053 2054 imalloc(&sopts, &dopts); 2055 2056 LOG("core.malloc.exit", "result: %p", ret); 2057 2058 return ret; 2059 } 2060 2061 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 2062 JEMALLOC_ATTR(nonnull(1)) 2063 je_posix_memalign(void **memptr, size_t alignment, size_t size) { 2064 int ret; 2065 static_opts_t sopts; 2066 dynamic_opts_t dopts; 2067 2068 LOG("core.posix_memalign.entry", "mem ptr: %p, alignment: %zu, " 2069 "size: %zu", memptr, alignment, size); 2070 2071 static_opts_init(&sopts); 2072 dynamic_opts_init(&dopts); 2073 2074 sopts.bump_empty_alloc = true; 2075 sopts.min_alignment = sizeof(void *); 2076 sopts.oom_string = 2077 "<jemalloc>: Error allocating aligned memory: out of memory\n"; 2078 sopts.invalid_alignment_string = 2079 "<jemalloc>: Error allocating aligned memory: invalid alignment\n"; 2080 2081 dopts.result = memptr; 2082 dopts.num_items = 1; 2083 dopts.item_size = size; 2084 dopts.alignment = alignment; 2085 2086 ret = imalloc(&sopts, &dopts); 2087 2088 LOG("core.posix_memalign.exit", "result: %d, alloc ptr: %p", ret, 2089 *memptr); 2090 2091 return ret; 2092 } 2093 2094 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2095 void JEMALLOC_NOTHROW * 2096 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(2) 2097 je_aligned_alloc(size_t alignment, size_t size) { 2098 void *ret; 2099 2100 static_opts_t sopts; 2101 dynamic_opts_t dopts; 2102 2103 LOG("core.aligned_alloc.entry", "alignment: %zu, size: %zu\n", 2104 alignment, size); 2105 2106 static_opts_init(&sopts); 2107 dynamic_opts_init(&dopts); 2108 2109 sopts.bump_empty_alloc = true; 2110 sopts.null_out_result_on_error = true; 2111 sopts.set_errno_on_error = true; 2112 sopts.min_alignment = 1; 2113 sopts.oom_string = 2114 "<jemalloc>: Error allocating aligned memory: out of memory\n"; 2115 sopts.invalid_alignment_string = 2116 "<jemalloc>: Error allocating aligned memory: invalid alignment\n"; 2117 2118 dopts.result = &ret; 2119 dopts.num_items = 1; 2120 dopts.item_size = size; 2121 dopts.alignment = alignment; 2122 2123 imalloc(&sopts, &dopts); 2124 2125 LOG("core.aligned_alloc.exit", "result: %p", ret); 2126 2127 return ret; 2128 } 2129 2130 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2131 void JEMALLOC_NOTHROW * 2132 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2) 2133 je_calloc(size_t num, size_t size) { 2134 void *ret; 2135 static_opts_t sopts; 2136 dynamic_opts_t dopts; 2137 2138 LOG("core.calloc.entry", "num: %zu, size: %zu\n", num, size); 2139 2140 static_opts_init(&sopts); 2141 dynamic_opts_init(&dopts); 2142 2143 sopts.may_overflow = true; 2144 sopts.bump_empty_alloc = true; 2145 sopts.null_out_result_on_error = true; 2146 sopts.set_errno_on_error = true; 2147 sopts.oom_string = "<jemalloc>: Error in calloc(): out of memory\n"; 2148 2149 dopts.result = &ret; 2150 dopts.num_items = num; 2151 dopts.item_size = size; 2152 dopts.zero = true; 2153 2154 imalloc(&sopts, &dopts); 2155 2156 LOG("core.calloc.exit", "result: %p", ret); 2157 2158 return ret; 2159 } 2160 2161 static void * 2162 irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, 2163 prof_tctx_t *tctx) { 2164 void *p; 2165 2166 if (tctx == NULL) { 2167 return NULL; 2168 } 2169 if (usize <= SMALL_MAXCLASS) { 2170 p = iralloc(tsd, old_ptr, old_usize, LARGE_MINCLASS, 0, false); 2171 if (p == NULL) { 2172 return NULL; 2173 } 2174 arena_prof_promote(tsd_tsdn(tsd), p, usize); 2175 } else { 2176 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); 2177 } 2178 2179 return p; 2180 } 2181 2182 JEMALLOC_ALWAYS_INLINE void * 2183 irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, 2184 alloc_ctx_t *alloc_ctx) { 2185 void *p; 2186 bool prof_active; 2187 prof_tctx_t *old_tctx, *tctx; 2188 2189 prof_active = prof_active_get_unlocked(); 2190 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr, alloc_ctx); 2191 tctx = prof_alloc_prep(tsd, usize, prof_active, true); 2192 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { 2193 p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx); 2194 } else { 2195 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); 2196 } 2197 if (unlikely(p == NULL)) { 2198 prof_alloc_rollback(tsd, tctx, true); 2199 return NULL; 2200 } 2201 prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize, 2202 old_tctx); 2203 2204 return p; 2205 } 2206 2207 JEMALLOC_ALWAYS_INLINE void 2208 ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) { 2209 if (!slow_path) { 2210 tsd_assert_fast(tsd); 2211 } 2212 check_entry_exit_locking(tsd_tsdn(tsd)); 2213 if (tsd_reentrancy_level_get(tsd) != 0) { 2214 assert(slow_path); 2215 } 2216 2217 assert(ptr != NULL); 2218 assert(malloc_initialized() || IS_INITIALIZER); 2219 2220 alloc_ctx_t alloc_ctx; 2221 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); 2222 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, 2223 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); 2224 assert(alloc_ctx.szind != NSIZES); 2225 2226 size_t usize; 2227 if (config_prof && opt_prof) { 2228 usize = sz_index2size(alloc_ctx.szind); 2229 prof_free(tsd, ptr, usize, &alloc_ctx); 2230 } else if (config_stats) { 2231 usize = sz_index2size(alloc_ctx.szind); 2232 } 2233 if (config_stats) { 2234 *tsd_thread_deallocatedp_get(tsd) += usize; 2235 } 2236 2237 if (likely(!slow_path)) { 2238 idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false, 2239 false); 2240 } else { 2241 idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false, 2242 true); 2243 } 2244 } 2245 2246 JEMALLOC_ALWAYS_INLINE void 2247 isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) { 2248 if (!slow_path) { 2249 tsd_assert_fast(tsd); 2250 } 2251 check_entry_exit_locking(tsd_tsdn(tsd)); 2252 if (tsd_reentrancy_level_get(tsd) != 0) { 2253 assert(slow_path); 2254 } 2255 2256 assert(ptr != NULL); 2257 assert(malloc_initialized() || IS_INITIALIZER); 2258 2259 alloc_ctx_t alloc_ctx, *ctx; 2260 if (!config_cache_oblivious && ((uintptr_t)ptr & PAGE_MASK) != 0) { 2261 /* 2262 * When cache_oblivious is disabled and ptr is not page aligned, 2263 * the allocation was not sampled -- usize can be used to 2264 * determine szind directly. 2265 */ 2266 alloc_ctx.szind = sz_size2index(usize); 2267 alloc_ctx.slab = true; 2268 ctx = &alloc_ctx; 2269 if (config_debug) { 2270 alloc_ctx_t dbg_ctx; 2271 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); 2272 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, 2273 rtree_ctx, (uintptr_t)ptr, true, &dbg_ctx.szind, 2274 &dbg_ctx.slab); 2275 assert(dbg_ctx.szind == alloc_ctx.szind); 2276 assert(dbg_ctx.slab == alloc_ctx.slab); 2277 } 2278 } else if (config_prof && opt_prof) { 2279 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); 2280 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, 2281 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); 2282 assert(alloc_ctx.szind == sz_size2index(usize)); 2283 ctx = &alloc_ctx; 2284 } else { 2285 ctx = NULL; 2286 } 2287 2288 if (config_prof && opt_prof) { 2289 prof_free(tsd, ptr, usize, ctx); 2290 } 2291 if (config_stats) { 2292 *tsd_thread_deallocatedp_get(tsd) += usize; 2293 } 2294 2295 if (likely(!slow_path)) { 2296 isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, false); 2297 } else { 2298 isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, true); 2299 } 2300 } 2301 2302 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2303 void JEMALLOC_NOTHROW * 2304 JEMALLOC_ALLOC_SIZE(2) 2305 je_realloc(void *ptr, size_t size) { 2306 void *ret; 2307 tsdn_t *tsdn JEMALLOC_CC_SILENCE_INIT(NULL); 2308 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 2309 size_t old_usize = 0; 2310 2311 LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size); 2312 2313 if (unlikely(size == 0)) { 2314 if (ptr != NULL) { 2315 /* realloc(ptr, 0) is equivalent to free(ptr). */ 2316 UTRACE(ptr, 0, 0); 2317 tcache_t *tcache; 2318 tsd_t *tsd = tsd_fetch(); 2319 if (tsd_reentrancy_level_get(tsd) == 0) { 2320 tcache = tcache_get(tsd); 2321 } else { 2322 tcache = NULL; 2323 } 2324 ifree(tsd, ptr, tcache, true); 2325 2326 LOG("core.realloc.exit", "result: %p", NULL); 2327 return NULL; 2328 } 2329 size = 1; 2330 } 2331 2332 if (likely(ptr != NULL)) { 2333 assert(malloc_initialized() || IS_INITIALIZER); 2334 tsd_t *tsd = tsd_fetch(); 2335 2336 check_entry_exit_locking(tsd_tsdn(tsd)); 2337 2338 alloc_ctx_t alloc_ctx; 2339 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); 2340 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, 2341 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); 2342 assert(alloc_ctx.szind != NSIZES); 2343 old_usize = sz_index2size(alloc_ctx.szind); 2344 assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); 2345 if (config_prof && opt_prof) { 2346 usize = sz_s2u(size); 2347 ret = unlikely(usize == 0 || usize > LARGE_MAXCLASS) ? 2348 NULL : irealloc_prof(tsd, ptr, old_usize, usize, 2349 &alloc_ctx); 2350 } else { 2351 if (config_stats) { 2352 usize = sz_s2u(size); 2353 } 2354 ret = iralloc(tsd, ptr, old_usize, size, 0, false); 2355 } 2356 tsdn = tsd_tsdn(tsd); 2357 } else { 2358 /* realloc(NULL, size) is equivalent to malloc(size). */ 2359 void *ret = je_malloc(size); 2360 LOG("core.realloc.exit", "result: %p", ret); 2361 return ret; 2362 } 2363 2364 if (unlikely(ret == NULL)) { 2365 if (config_xmalloc && unlikely(opt_xmalloc)) { 2366 malloc_write("<jemalloc>: Error in realloc(): " 2367 "out of memory\n"); 2368 abort(); 2369 } 2370 set_errno(ENOMEM); 2371 } 2372 if (config_stats && likely(ret != NULL)) { 2373 tsd_t *tsd; 2374 2375 assert(usize == isalloc(tsdn, ret)); 2376 tsd = tsdn_tsd(tsdn); 2377 *tsd_thread_allocatedp_get(tsd) += usize; 2378 *tsd_thread_deallocatedp_get(tsd) += old_usize; 2379 } 2380 UTRACE(ptr, size, ret); 2381 check_entry_exit_locking(tsdn); 2382 2383 LOG("core.realloc.exit", "result: %p", ret); 2384 return ret; 2385 } 2386 2387 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 2388 je_free(void *ptr) { 2389 LOG("core.free.entry", "ptr: %p", ptr); 2390 2391 UTRACE(ptr, 0, 0); 2392 if (likely(ptr != NULL)) { 2393 /* 2394 * We avoid setting up tsd fully (e.g. tcache, arena binding) 2395 * based on only free() calls -- other activities trigger the 2396 * minimal to full transition. This is because free() may 2397 * happen during thread shutdown after tls deallocation: if a 2398 * thread never had any malloc activities until then, a 2399 * fully-setup tsd won't be destructed properly. 2400 */ 2401 tsd_t *tsd = tsd_fetch_min(); 2402 check_entry_exit_locking(tsd_tsdn(tsd)); 2403 2404 tcache_t *tcache; 2405 if (likely(tsd_fast(tsd))) { 2406 tsd_assert_fast(tsd); 2407 /* Unconditionally get tcache ptr on fast path. */ 2408 tcache = tsd_tcachep_get(tsd); 2409 ifree(tsd, ptr, tcache, false); 2410 } else { 2411 if (likely(tsd_reentrancy_level_get(tsd) == 0)) { 2412 tcache = tcache_get(tsd); 2413 } else { 2414 tcache = NULL; 2415 } 2416 ifree(tsd, ptr, tcache, true); 2417 } 2418 check_entry_exit_locking(tsd_tsdn(tsd)); 2419 } 2420 LOG("core.free.exit", ""); 2421 } 2422 2423 /* 2424 * End malloc(3)-compatible functions. 2425 */ 2426 /******************************************************************************/ 2427 /* 2428 * Begin non-standard override functions. 2429 */ 2430 2431 #ifdef JEMALLOC_OVERRIDE_MEMALIGN 2432 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2433 void JEMALLOC_NOTHROW * 2434 JEMALLOC_ATTR(malloc) 2435 je_memalign(size_t alignment, size_t size) { 2436 void *ret; 2437 static_opts_t sopts; 2438 dynamic_opts_t dopts; 2439 2440 LOG("core.memalign.entry", "alignment: %zu, size: %zu\n", alignment, 2441 size); 2442 2443 static_opts_init(&sopts); 2444 dynamic_opts_init(&dopts); 2445 2446 sopts.bump_empty_alloc = true; 2447 sopts.min_alignment = 1; 2448 sopts.oom_string = 2449 "<jemalloc>: Error allocating aligned memory: out of memory\n"; 2450 sopts.invalid_alignment_string = 2451 "<jemalloc>: Error allocating aligned memory: invalid alignment\n"; 2452 sopts.null_out_result_on_error = true; 2453 2454 dopts.result = &ret; 2455 dopts.num_items = 1; 2456 dopts.item_size = size; 2457 dopts.alignment = alignment; 2458 2459 imalloc(&sopts, &dopts); 2460 2461 LOG("core.memalign.exit", "result: %p", ret); 2462 return ret; 2463 } 2464 #endif 2465 2466 #ifdef JEMALLOC_OVERRIDE_VALLOC 2467 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2468 void JEMALLOC_NOTHROW * 2469 JEMALLOC_ATTR(malloc) 2470 je_valloc(size_t size) { 2471 void *ret; 2472 2473 static_opts_t sopts; 2474 dynamic_opts_t dopts; 2475 2476 LOG("core.valloc.entry", "size: %zu\n", size); 2477 2478 static_opts_init(&sopts); 2479 dynamic_opts_init(&dopts); 2480 2481 sopts.bump_empty_alloc = true; 2482 sopts.null_out_result_on_error = true; 2483 sopts.min_alignment = PAGE; 2484 sopts.oom_string = 2485 "<jemalloc>: Error allocating aligned memory: out of memory\n"; 2486 sopts.invalid_alignment_string = 2487 "<jemalloc>: Error allocating aligned memory: invalid alignment\n"; 2488 2489 dopts.result = &ret; 2490 dopts.num_items = 1; 2491 dopts.item_size = size; 2492 dopts.alignment = PAGE; 2493 2494 imalloc(&sopts, &dopts); 2495 2496 LOG("core.valloc.exit", "result: %p\n", ret); 2497 return ret; 2498 } 2499 #endif 2500 2501 #if defined(JEMALLOC_IS_MALLOC) && defined(JEMALLOC_GLIBC_MALLOC_HOOK) 2502 /* 2503 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible 2504 * to inconsistently reference libc's malloc(3)-compatible functions 2505 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541). 2506 * 2507 * These definitions interpose hooks in glibc. The functions are actually 2508 * passed an extra argument for the caller return address, which will be 2509 * ignored. 2510 */ 2511 JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free; 2512 JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc; 2513 JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc; 2514 # ifdef JEMALLOC_GLIBC_MEMALIGN_HOOK 2515 JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) = 2516 je_memalign; 2517 # endif 2518 2519 # ifdef CPU_COUNT 2520 /* 2521 * To enable static linking with glibc, the libc specific malloc interface must 2522 * be implemented also, so none of glibc's malloc.o functions are added to the 2523 * link. 2524 */ 2525 # define ALIAS(je_fn) __attribute__((alias (#je_fn), used)) 2526 /* To force macro expansion of je_ prefix before stringification. */ 2527 # define PREALIAS(je_fn) ALIAS(je_fn) 2528 # ifdef JEMALLOC_OVERRIDE___LIBC_CALLOC 2529 void *__libc_calloc(size_t n, size_t size) PREALIAS(je_calloc); 2530 # endif 2531 # ifdef JEMALLOC_OVERRIDE___LIBC_FREE 2532 void __libc_free(void* ptr) PREALIAS(je_free); 2533 # endif 2534 # ifdef JEMALLOC_OVERRIDE___LIBC_MALLOC 2535 void *__libc_malloc(size_t size) PREALIAS(je_malloc); 2536 # endif 2537 # ifdef JEMALLOC_OVERRIDE___LIBC_MEMALIGN 2538 void *__libc_memalign(size_t align, size_t s) PREALIAS(je_memalign); 2539 # endif 2540 # ifdef JEMALLOC_OVERRIDE___LIBC_REALLOC 2541 void *__libc_realloc(void* ptr, size_t size) PREALIAS(je_realloc); 2542 # endif 2543 # ifdef JEMALLOC_OVERRIDE___LIBC_VALLOC 2544 void *__libc_valloc(size_t size) PREALIAS(je_valloc); 2545 # endif 2546 # ifdef JEMALLOC_OVERRIDE___POSIX_MEMALIGN 2547 int __posix_memalign(void** r, size_t a, size_t s) PREALIAS(je_posix_memalign); 2548 # endif 2549 # undef PREALIAS 2550 # undef ALIAS 2551 # endif 2552 #endif 2553 2554 /* 2555 * End non-standard override functions. 2556 */ 2557 /******************************************************************************/ 2558 /* 2559 * Begin non-standard functions. 2560 */ 2561 2562 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2563 void JEMALLOC_NOTHROW * 2564 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) 2565 je_mallocx(size_t size, int flags) { 2566 void *ret; 2567 static_opts_t sopts; 2568 dynamic_opts_t dopts; 2569 2570 LOG("core.mallocx.entry", "size: %zu, flags: %d", size, flags); 2571 2572 static_opts_init(&sopts); 2573 dynamic_opts_init(&dopts); 2574 2575 sopts.assert_nonempty_alloc = true; 2576 sopts.null_out_result_on_error = true; 2577 sopts.oom_string = "<jemalloc>: Error in mallocx(): out of memory\n"; 2578 2579 dopts.result = &ret; 2580 dopts.num_items = 1; 2581 dopts.item_size = size; 2582 if (unlikely(flags != 0)) { 2583 if ((flags & MALLOCX_LG_ALIGN_MASK) != 0) { 2584 dopts.alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags); 2585 } 2586 2587 dopts.zero = MALLOCX_ZERO_GET(flags); 2588 2589 if ((flags & MALLOCX_TCACHE_MASK) != 0) { 2590 if ((flags & MALLOCX_TCACHE_MASK) 2591 == MALLOCX_TCACHE_NONE) { 2592 dopts.tcache_ind = TCACHE_IND_NONE; 2593 } else { 2594 dopts.tcache_ind = MALLOCX_TCACHE_GET(flags); 2595 } 2596 } else { 2597 dopts.tcache_ind = TCACHE_IND_AUTOMATIC; 2598 } 2599 2600 if ((flags & MALLOCX_ARENA_MASK) != 0) 2601 dopts.arena_ind = MALLOCX_ARENA_GET(flags); 2602 } 2603 2604 imalloc(&sopts, &dopts); 2605 2606 LOG("core.mallocx.exit", "result: %p", ret); 2607 return ret; 2608 } 2609 2610 static void * 2611 irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize, 2612 size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena, 2613 prof_tctx_t *tctx) { 2614 void *p; 2615 2616 if (tctx == NULL) { 2617 return NULL; 2618 } 2619 if (usize <= SMALL_MAXCLASS) { 2620 p = iralloct(tsdn, old_ptr, old_usize, LARGE_MINCLASS, 2621 alignment, zero, tcache, arena); 2622 if (p == NULL) { 2623 return NULL; 2624 } 2625 arena_prof_promote(tsdn, p, usize); 2626 } else { 2627 p = iralloct(tsdn, old_ptr, old_usize, usize, alignment, zero, 2628 tcache, arena); 2629 } 2630 2631 return p; 2632 } 2633 2634 JEMALLOC_ALWAYS_INLINE void * 2635 irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, 2636 size_t alignment, size_t *usize, bool zero, tcache_t *tcache, 2637 arena_t *arena, alloc_ctx_t *alloc_ctx) { 2638 void *p; 2639 bool prof_active; 2640 prof_tctx_t *old_tctx, *tctx; 2641 2642 prof_active = prof_active_get_unlocked(); 2643 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr, alloc_ctx); 2644 tctx = prof_alloc_prep(tsd, *usize, prof_active, false); 2645 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { 2646 p = irallocx_prof_sample(tsd_tsdn(tsd), old_ptr, old_usize, 2647 *usize, alignment, zero, tcache, arena, tctx); 2648 } else { 2649 p = iralloct(tsd_tsdn(tsd), old_ptr, old_usize, size, alignment, 2650 zero, tcache, arena); 2651 } 2652 if (unlikely(p == NULL)) { 2653 prof_alloc_rollback(tsd, tctx, false); 2654 return NULL; 2655 } 2656 2657 if (p == old_ptr && alignment != 0) { 2658 /* 2659 * The allocation did not move, so it is possible that the size 2660 * class is smaller than would guarantee the requested 2661 * alignment, and that the alignment constraint was 2662 * serendipitously satisfied. Additionally, old_usize may not 2663 * be the same as the current usize because of in-place large 2664 * reallocation. Therefore, query the actual value of usize. 2665 */ 2666 *usize = isalloc(tsd_tsdn(tsd), p); 2667 } 2668 prof_realloc(tsd, p, *usize, tctx, prof_active, false, old_ptr, 2669 old_usize, old_tctx); 2670 2671 return p; 2672 } 2673 2674 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2675 void JEMALLOC_NOTHROW * 2676 JEMALLOC_ALLOC_SIZE(2) 2677 je_rallocx(void *ptr, size_t size, int flags) { 2678 void *p; 2679 tsd_t *tsd; 2680 size_t usize; 2681 size_t old_usize; 2682 size_t alignment = MALLOCX_ALIGN_GET(flags); 2683 bool zero = flags & MALLOCX_ZERO; 2684 arena_t *arena; 2685 tcache_t *tcache; 2686 2687 LOG("core.rallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr, 2688 size, flags); 2689 2690 2691 assert(ptr != NULL); 2692 assert(size != 0); 2693 assert(malloc_initialized() || IS_INITIALIZER); 2694 tsd = tsd_fetch(); 2695 check_entry_exit_locking(tsd_tsdn(tsd)); 2696 2697 if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { 2698 unsigned arena_ind = MALLOCX_ARENA_GET(flags); 2699 arena = arena_get(tsd_tsdn(tsd), arena_ind, true); 2700 if (unlikely(arena == NULL)) { 2701 goto label_oom; 2702 } 2703 } else { 2704 arena = NULL; 2705 } 2706 2707 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 2708 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) { 2709 tcache = NULL; 2710 } else { 2711 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2712 } 2713 } else { 2714 tcache = tcache_get(tsd); 2715 } 2716 2717 alloc_ctx_t alloc_ctx; 2718 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); 2719 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, 2720 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); 2721 assert(alloc_ctx.szind != NSIZES); 2722 old_usize = sz_index2size(alloc_ctx.szind); 2723 assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); 2724 if (config_prof && opt_prof) { 2725 usize = (alignment == 0) ? 2726 sz_s2u(size) : sz_sa2u(size, alignment); 2727 if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { 2728 goto label_oom; 2729 } 2730 p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize, 2731 zero, tcache, arena, &alloc_ctx); 2732 if (unlikely(p == NULL)) { 2733 goto label_oom; 2734 } 2735 } else { 2736 p = iralloct(tsd_tsdn(tsd), ptr, old_usize, size, alignment, 2737 zero, tcache, arena); 2738 if (unlikely(p == NULL)) { 2739 goto label_oom; 2740 } 2741 if (config_stats) { 2742 usize = isalloc(tsd_tsdn(tsd), p); 2743 } 2744 } 2745 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0)); 2746 2747 if (config_stats) { 2748 *tsd_thread_allocatedp_get(tsd) += usize; 2749 *tsd_thread_deallocatedp_get(tsd) += old_usize; 2750 } 2751 UTRACE(ptr, size, p); 2752 check_entry_exit_locking(tsd_tsdn(tsd)); 2753 2754 LOG("core.rallocx.exit", "result: %p", p); 2755 return p; 2756 label_oom: 2757 if (config_xmalloc && unlikely(opt_xmalloc)) { 2758 malloc_write("<jemalloc>: Error in rallocx(): out of memory\n"); 2759 abort(); 2760 } 2761 UTRACE(ptr, size, 0); 2762 check_entry_exit_locking(tsd_tsdn(tsd)); 2763 2764 LOG("core.rallocx.exit", "result: %p", NULL); 2765 return NULL; 2766 } 2767 2768 JEMALLOC_ALWAYS_INLINE size_t 2769 ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size, 2770 size_t extra, size_t alignment, bool zero) { 2771 size_t usize; 2772 2773 if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero)) { 2774 return old_usize; 2775 } 2776 usize = isalloc(tsdn, ptr); 2777 2778 return usize; 2779 } 2780 2781 static size_t 2782 ixallocx_prof_sample(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size, 2783 size_t extra, size_t alignment, bool zero, prof_tctx_t *tctx) { 2784 size_t usize; 2785 2786 if (tctx == NULL) { 2787 return old_usize; 2788 } 2789 usize = ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment, 2790 zero); 2791 2792 return usize; 2793 } 2794 2795 JEMALLOC_ALWAYS_INLINE size_t 2796 ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, 2797 size_t extra, size_t alignment, bool zero, alloc_ctx_t *alloc_ctx) { 2798 size_t usize_max, usize; 2799 bool prof_active; 2800 prof_tctx_t *old_tctx, *tctx; 2801 2802 prof_active = prof_active_get_unlocked(); 2803 old_tctx = prof_tctx_get(tsd_tsdn(tsd), ptr, alloc_ctx); 2804 /* 2805 * usize isn't knowable before ixalloc() returns when extra is non-zero. 2806 * Therefore, compute its maximum possible value and use that in 2807 * prof_alloc_prep() to decide whether to capture a backtrace. 2808 * prof_realloc() will use the actual usize to decide whether to sample. 2809 */ 2810 if (alignment == 0) { 2811 usize_max = sz_s2u(size+extra); 2812 assert(usize_max > 0 && usize_max <= LARGE_MAXCLASS); 2813 } else { 2814 usize_max = sz_sa2u(size+extra, alignment); 2815 if (unlikely(usize_max == 0 || usize_max > LARGE_MAXCLASS)) { 2816 /* 2817 * usize_max is out of range, and chances are that 2818 * allocation will fail, but use the maximum possible 2819 * value and carry on with prof_alloc_prep(), just in 2820 * case allocation succeeds. 2821 */ 2822 usize_max = LARGE_MAXCLASS; 2823 } 2824 } 2825 tctx = prof_alloc_prep(tsd, usize_max, prof_active, false); 2826 2827 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { 2828 usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize, 2829 size, extra, alignment, zero, tctx); 2830 } else { 2831 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size, 2832 extra, alignment, zero); 2833 } 2834 if (usize == old_usize) { 2835 prof_alloc_rollback(tsd, tctx, false); 2836 return usize; 2837 } 2838 prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize, 2839 old_tctx); 2840 2841 return usize; 2842 } 2843 2844 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 2845 je_xallocx(void *ptr, size_t size, size_t extra, int flags) { 2846 tsd_t *tsd; 2847 size_t usize, old_usize; 2848 size_t alignment = MALLOCX_ALIGN_GET(flags); 2849 bool zero = flags & MALLOCX_ZERO; 2850 2851 LOG("core.xallocx.entry", "ptr: %p, size: %zu, extra: %zu, " 2852 "flags: %d", ptr, size, extra, flags); 2853 2854 assert(ptr != NULL); 2855 assert(size != 0); 2856 assert(SIZE_T_MAX - size >= extra); 2857 assert(malloc_initialized() || IS_INITIALIZER); 2858 tsd = tsd_fetch(); 2859 check_entry_exit_locking(tsd_tsdn(tsd)); 2860 2861 alloc_ctx_t alloc_ctx; 2862 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); 2863 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, 2864 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); 2865 assert(alloc_ctx.szind != NSIZES); 2866 old_usize = sz_index2size(alloc_ctx.szind); 2867 assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); 2868 /* 2869 * The API explicitly absolves itself of protecting against (size + 2870 * extra) numerical overflow, but we may need to clamp extra to avoid 2871 * exceeding LARGE_MAXCLASS. 2872 * 2873 * Ordinarily, size limit checking is handled deeper down, but here we 2874 * have to check as part of (size + extra) clamping, since we need the 2875 * clamped value in the above helper functions. 2876 */ 2877 if (unlikely(size > LARGE_MAXCLASS)) { 2878 usize = old_usize; 2879 goto label_not_resized; 2880 } 2881 if (unlikely(LARGE_MAXCLASS - size < extra)) { 2882 extra = LARGE_MAXCLASS - size; 2883 } 2884 2885 if (config_prof && opt_prof) { 2886 usize = ixallocx_prof(tsd, ptr, old_usize, size, extra, 2887 alignment, zero, &alloc_ctx); 2888 } else { 2889 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size, 2890 extra, alignment, zero); 2891 } 2892 if (unlikely(usize == old_usize)) { 2893 goto label_not_resized; 2894 } 2895 2896 if (config_stats) { 2897 *tsd_thread_allocatedp_get(tsd) += usize; 2898 *tsd_thread_deallocatedp_get(tsd) += old_usize; 2899 } 2900 label_not_resized: 2901 UTRACE(ptr, size, ptr); 2902 check_entry_exit_locking(tsd_tsdn(tsd)); 2903 2904 LOG("core.xallocx.exit", "result: %zu", usize); 2905 return usize; 2906 } 2907 2908 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 2909 JEMALLOC_ATTR(pure) 2910 je_sallocx(const void *ptr, UNUSED int flags) { 2911 size_t usize; 2912 tsdn_t *tsdn; 2913 2914 LOG("core.sallocx.entry", "ptr: %p, flags: %d", ptr, flags); 2915 2916 assert(malloc_initialized() || IS_INITIALIZER); 2917 assert(ptr != NULL); 2918 2919 tsdn = tsdn_fetch(); 2920 check_entry_exit_locking(tsdn); 2921 2922 if (config_debug || force_ivsalloc) { 2923 usize = ivsalloc(tsdn, ptr); 2924 assert(force_ivsalloc || usize != 0); 2925 } else { 2926 usize = isalloc(tsdn, ptr); 2927 } 2928 2929 check_entry_exit_locking(tsdn); 2930 2931 LOG("core.sallocx.exit", "result: %zu", usize); 2932 return usize; 2933 } 2934 2935 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 2936 je_dallocx(void *ptr, int flags) { 2937 LOG("core.dallocx.entry", "ptr: %p, flags: %d", ptr, flags); 2938 2939 assert(ptr != NULL); 2940 assert(malloc_initialized() || IS_INITIALIZER); 2941 2942 tsd_t *tsd = tsd_fetch(); 2943 bool fast = tsd_fast(tsd); 2944 check_entry_exit_locking(tsd_tsdn(tsd)); 2945 2946 tcache_t *tcache; 2947 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 2948 /* Not allowed to be reentrant and specify a custom tcache. */ 2949 assert(tsd_reentrancy_level_get(tsd) == 0); 2950 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) { 2951 tcache = NULL; 2952 } else { 2953 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2954 } 2955 } else { 2956 if (likely(fast)) { 2957 tcache = tsd_tcachep_get(tsd); 2958 assert(tcache == tcache_get(tsd)); 2959 } else { 2960 if (likely(tsd_reentrancy_level_get(tsd) == 0)) { 2961 tcache = tcache_get(tsd); 2962 } else { 2963 tcache = NULL; 2964 } 2965 } 2966 } 2967 2968 UTRACE(ptr, 0, 0); 2969 if (likely(fast)) { 2970 tsd_assert_fast(tsd); 2971 ifree(tsd, ptr, tcache, false); 2972 } else { 2973 ifree(tsd, ptr, tcache, true); 2974 } 2975 check_entry_exit_locking(tsd_tsdn(tsd)); 2976 2977 LOG("core.dallocx.exit", ""); 2978 } 2979 2980 JEMALLOC_ALWAYS_INLINE size_t 2981 inallocx(tsdn_t *tsdn, size_t size, int flags) { 2982 check_entry_exit_locking(tsdn); 2983 2984 size_t usize; 2985 if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) { 2986 usize = sz_s2u(size); 2987 } else { 2988 usize = sz_sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags)); 2989 } 2990 check_entry_exit_locking(tsdn); 2991 return usize; 2992 } 2993 2994 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 2995 je_sdallocx(void *ptr, size_t size, int flags) { 2996 assert(ptr != NULL); 2997 assert(malloc_initialized() || IS_INITIALIZER); 2998 2999 LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr, 3000 size, flags); 3001 3002 tsd_t *tsd = tsd_fetch(); 3003 bool fast = tsd_fast(tsd); 3004 size_t usize = inallocx(tsd_tsdn(tsd), size, flags); 3005 assert(usize == isalloc(tsd_tsdn(tsd), ptr)); 3006 check_entry_exit_locking(tsd_tsdn(tsd)); 3007 3008 tcache_t *tcache; 3009 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 3010 /* Not allowed to be reentrant and specify a custom tcache. */ 3011 assert(tsd_reentrancy_level_get(tsd) == 0); 3012 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) { 3013 tcache = NULL; 3014 } else { 3015 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 3016 } 3017 } else { 3018 if (likely(fast)) { 3019 tcache = tsd_tcachep_get(tsd); 3020 assert(tcache == tcache_get(tsd)); 3021 } else { 3022 if (likely(tsd_reentrancy_level_get(tsd) == 0)) { 3023 tcache = tcache_get(tsd); 3024 } else { 3025 tcache = NULL; 3026 } 3027 } 3028 } 3029 3030 UTRACE(ptr, 0, 0); 3031 if (likely(fast)) { 3032 tsd_assert_fast(tsd); 3033 isfree(tsd, ptr, usize, tcache, false); 3034 } else { 3035 isfree(tsd, ptr, usize, tcache, true); 3036 } 3037 check_entry_exit_locking(tsd_tsdn(tsd)); 3038 3039 LOG("core.sdallocx.exit", ""); 3040 } 3041 3042 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 3043 JEMALLOC_ATTR(pure) 3044 je_nallocx(size_t size, int flags) { 3045 size_t usize; 3046 tsdn_t *tsdn; 3047 3048 assert(size != 0); 3049 3050 if (unlikely(malloc_init())) { 3051 LOG("core.nallocx.exit", "result: %zu", ZU(0)); 3052 return 0; 3053 } 3054 3055 tsdn = tsdn_fetch(); 3056 check_entry_exit_locking(tsdn); 3057 3058 usize = inallocx(tsdn, size, flags); 3059 if (unlikely(usize > LARGE_MAXCLASS)) { 3060 LOG("core.nallocx.exit", "result: %zu", ZU(0)); 3061 return 0; 3062 } 3063 3064 check_entry_exit_locking(tsdn); 3065 LOG("core.nallocx.exit", "result: %zu", usize); 3066 return usize; 3067 } 3068 3069 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 3070 je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, 3071 size_t newlen) { 3072 int ret; 3073 tsd_t *tsd; 3074 3075 LOG("core.mallctl.entry", "name: %s", name); 3076 3077 if (unlikely(malloc_init())) { 3078 LOG("core.mallctl.exit", "result: %d", EAGAIN); 3079 return EAGAIN; 3080 } 3081 3082 tsd = tsd_fetch(); 3083 check_entry_exit_locking(tsd_tsdn(tsd)); 3084 ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen); 3085 check_entry_exit_locking(tsd_tsdn(tsd)); 3086 3087 LOG("core.mallctl.exit", "result: %d", ret); 3088 return ret; 3089 } 3090 3091 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 3092 je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) { 3093 int ret; 3094 3095 LOG("core.mallctlnametomib.entry", "name: %s", name); 3096 3097 if (unlikely(malloc_init())) { 3098 LOG("core.mallctlnametomib.exit", "result: %d", EAGAIN); 3099 return EAGAIN; 3100 } 3101 3102 tsd_t *tsd = tsd_fetch(); 3103 check_entry_exit_locking(tsd_tsdn(tsd)); 3104 ret = ctl_nametomib(tsd, name, mibp, miblenp); 3105 check_entry_exit_locking(tsd_tsdn(tsd)); 3106 3107 LOG("core.mallctlnametomib.exit", "result: %d", ret); 3108 return ret; 3109 } 3110 3111 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 3112 je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 3113 void *newp, size_t newlen) { 3114 int ret; 3115 tsd_t *tsd; 3116 3117 LOG("core.mallctlbymib.entry", ""); 3118 3119 if (unlikely(malloc_init())) { 3120 LOG("core.mallctlbymib.exit", "result: %d", EAGAIN); 3121 return EAGAIN; 3122 } 3123 3124 tsd = tsd_fetch(); 3125 check_entry_exit_locking(tsd_tsdn(tsd)); 3126 ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen); 3127 check_entry_exit_locking(tsd_tsdn(tsd)); 3128 LOG("core.mallctlbymib.exit", "result: %d", ret); 3129 return ret; 3130 } 3131 3132 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 3133 je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, 3134 const char *opts) { 3135 tsdn_t *tsdn; 3136 3137 LOG("core.malloc_stats_print.entry", ""); 3138 3139 tsdn = tsdn_fetch(); 3140 check_entry_exit_locking(tsdn); 3141 stats_print(write_cb, cbopaque, opts); 3142 check_entry_exit_locking(tsdn); 3143 LOG("core.malloc_stats_print.exit", ""); 3144 } 3145 3146 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 3147 je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) { 3148 size_t ret; 3149 tsdn_t *tsdn; 3150 3151 LOG("core.malloc_usable_size.entry", "ptr: %p", ptr); 3152 3153 assert(malloc_initialized() || IS_INITIALIZER); 3154 3155 tsdn = tsdn_fetch(); 3156 check_entry_exit_locking(tsdn); 3157 3158 if (unlikely(ptr == NULL)) { 3159 ret = 0; 3160 } else { 3161 if (config_debug || force_ivsalloc) { 3162 ret = ivsalloc(tsdn, ptr); 3163 assert(force_ivsalloc || ret != 0); 3164 } else { 3165 ret = isalloc(tsdn, ptr); 3166 } 3167 } 3168 3169 check_entry_exit_locking(tsdn); 3170 LOG("core.malloc_usable_size.exit", "result: %zu", ret); 3171 return ret; 3172 } 3173 3174 /* 3175 * End non-standard functions. 3176 */ 3177 /******************************************************************************/ 3178 /* 3179 * The following functions are used by threading libraries for protection of 3180 * malloc during fork(). 3181 */ 3182 3183 /* 3184 * If an application creates a thread before doing any allocation in the main 3185 * thread, then calls fork(2) in the main thread followed by memory allocation 3186 * in the child process, a race can occur that results in deadlock within the 3187 * child: the main thread may have forked while the created thread had 3188 * partially initialized the allocator. Ordinarily jemalloc prevents 3189 * fork/malloc races via the following functions it registers during 3190 * initialization using pthread_atfork(), but of course that does no good if 3191 * the allocator isn't fully initialized at fork time. The following library 3192 * constructor is a partial solution to this problem. It may still be possible 3193 * to trigger the deadlock described above, but doing so would involve forking 3194 * via a library constructor that runs before jemalloc's runs. 3195 */ 3196 #ifndef JEMALLOC_JET 3197 JEMALLOC_ATTR(constructor) 3198 static void 3199 jemalloc_constructor(void) { 3200 malloc_init(); 3201 } 3202 #endif 3203 3204 #ifndef JEMALLOC_MUTEX_INIT_CB 3205 void 3206 jemalloc_prefork(void) 3207 #else 3208 JEMALLOC_EXPORT void 3209 _malloc_prefork(void) 3210 #endif 3211 { 3212 tsd_t *tsd; 3213 unsigned i, j, narenas; 3214 arena_t *arena; 3215 3216 #ifdef JEMALLOC_MUTEX_INIT_CB 3217 if (!malloc_initialized()) { 3218 return; 3219 } 3220 #endif 3221 assert(malloc_initialized()); 3222 3223 tsd = tsd_fetch(); 3224 3225 narenas = narenas_total_get(); 3226 3227 witness_prefork(tsd_witness_tsdp_get(tsd)); 3228 /* Acquire all mutexes in a safe order. */ 3229 ctl_prefork(tsd_tsdn(tsd)); 3230 tcache_prefork(tsd_tsdn(tsd)); 3231 malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock); 3232 if (have_background_thread) { 3233 background_thread_prefork0(tsd_tsdn(tsd)); 3234 } 3235 prof_prefork0(tsd_tsdn(tsd)); 3236 if (have_background_thread) { 3237 background_thread_prefork1(tsd_tsdn(tsd)); 3238 } 3239 /* Break arena prefork into stages to preserve lock order. */ 3240 for (i = 0; i < 8; i++) { 3241 for (j = 0; j < narenas; j++) { 3242 if ((arena = arena_get(tsd_tsdn(tsd), j, false)) != 3243 NULL) { 3244 switch (i) { 3245 case 0: 3246 arena_prefork0(tsd_tsdn(tsd), arena); 3247 break; 3248 case 1: 3249 arena_prefork1(tsd_tsdn(tsd), arena); 3250 break; 3251 case 2: 3252 arena_prefork2(tsd_tsdn(tsd), arena); 3253 break; 3254 case 3: 3255 arena_prefork3(tsd_tsdn(tsd), arena); 3256 break; 3257 case 4: 3258 arena_prefork4(tsd_tsdn(tsd), arena); 3259 break; 3260 case 5: 3261 arena_prefork5(tsd_tsdn(tsd), arena); 3262 break; 3263 case 6: 3264 arena_prefork6(tsd_tsdn(tsd), arena); 3265 break; 3266 case 7: 3267 arena_prefork7(tsd_tsdn(tsd), arena); 3268 break; 3269 default: not_reached(); 3270 } 3271 } 3272 } 3273 } 3274 prof_prefork1(tsd_tsdn(tsd)); 3275 } 3276 3277 #ifndef JEMALLOC_MUTEX_INIT_CB 3278 void 3279 jemalloc_postfork_parent(void) 3280 #else 3281 JEMALLOC_EXPORT void 3282 _malloc_postfork(void) 3283 #endif 3284 { 3285 tsd_t *tsd; 3286 unsigned i, narenas; 3287 3288 #ifdef JEMALLOC_MUTEX_INIT_CB 3289 if (!malloc_initialized()) { 3290 return; 3291 } 3292 #endif 3293 assert(malloc_initialized()); 3294 3295 tsd = tsd_fetch(); 3296 3297 witness_postfork_parent(tsd_witness_tsdp_get(tsd)); 3298 /* Release all mutexes, now that fork() has completed. */ 3299 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 3300 arena_t *arena; 3301 3302 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) { 3303 arena_postfork_parent(tsd_tsdn(tsd), arena); 3304 } 3305 } 3306 prof_postfork_parent(tsd_tsdn(tsd)); 3307 if (have_background_thread) { 3308 background_thread_postfork_parent(tsd_tsdn(tsd)); 3309 } 3310 malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock); 3311 tcache_postfork_parent(tsd_tsdn(tsd)); 3312 ctl_postfork_parent(tsd_tsdn(tsd)); 3313 } 3314 3315 void 3316 jemalloc_postfork_child(void) { 3317 tsd_t *tsd; 3318 unsigned i, narenas; 3319 3320 assert(malloc_initialized()); 3321 3322 tsd = tsd_fetch(); 3323 3324 witness_postfork_child(tsd_witness_tsdp_get(tsd)); 3325 /* Release all mutexes, now that fork() has completed. */ 3326 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 3327 arena_t *arena; 3328 3329 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) { 3330 arena_postfork_child(tsd_tsdn(tsd), arena); 3331 } 3332 } 3333 prof_postfork_child(tsd_tsdn(tsd)); 3334 if (have_background_thread) { 3335 background_thread_postfork_child(tsd_tsdn(tsd)); 3336 } 3337 malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock); 3338 tcache_postfork_child(tsd_tsdn(tsd)); 3339 ctl_postfork_child(tsd_tsdn(tsd)); 3340 } 3341 3342 /******************************************************************************/ 3343 3344 #if defined(__BIONIC__) && !defined(JEMALLOC_JET) 3345 #include "android_je_iterate.c" 3346 #include "android_je_mallinfo.c" 3347 #endif 3348