1 #define JEMALLOC_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 4 /******************************************************************************/ 5 /* Data. */ 6 7 /* Runtime configuration options. */ 8 const char *je_malloc_conf 9 #ifndef _WIN32 10 JEMALLOC_ATTR(weak) 11 #endif 12 ; 13 bool opt_abort = 14 #ifdef JEMALLOC_DEBUG 15 true 16 #else 17 false 18 #endif 19 ; 20 const char *opt_junk = 21 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 22 "true" 23 #else 24 "false" 25 #endif 26 ; 27 bool opt_junk_alloc = 28 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 29 true 30 #else 31 false 32 #endif 33 ; 34 bool opt_junk_free = 35 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 36 true 37 #else 38 false 39 #endif 40 ; 41 42 size_t opt_quarantine = ZU(0); 43 bool opt_redzone = false; 44 bool opt_utrace = false; 45 bool opt_xmalloc = false; 46 bool opt_zero = false; 47 unsigned opt_narenas = 0; 48 49 /* Initialized to true if the process is running inside Valgrind. */ 50 bool in_valgrind; 51 52 unsigned ncpus; 53 54 /* Protects arenas initialization. */ 55 static malloc_mutex_t arenas_lock; 56 /* 57 * Arenas that are used to service external requests. Not all elements of the 58 * arenas array are necessarily used; arenas are created lazily as needed. 59 * 60 * arenas[0..narenas_auto) are used for automatic multiplexing of threads and 61 * arenas. arenas[narenas_auto..narenas_total) are only used if the application 62 * takes some action to create them and allocate from them. 63 */ 64 arena_t **arenas; 65 static unsigned narenas_total; /* Use narenas_total_*(). */ 66 static arena_t *a0; /* arenas[0]; read-only after initialization. */ 67 unsigned narenas_auto; /* Read-only after initialization. */ 68 69 typedef enum { 70 malloc_init_uninitialized = 3, 71 malloc_init_a0_initialized = 2, 72 malloc_init_recursible = 1, 73 malloc_init_initialized = 0 /* Common case --> jnz. */ 74 } malloc_init_t; 75 static malloc_init_t malloc_init_state = malloc_init_uninitialized; 76 77 /* False should be the common case. Set to true to trigger initialization. */ 78 static bool malloc_slow = true; 79 80 /* When malloc_slow is true, set the corresponding bits for sanity check. */ 81 enum { 82 flag_opt_junk_alloc = (1U), 83 flag_opt_junk_free = (1U << 1), 84 flag_opt_quarantine = (1U << 2), 85 flag_opt_zero = (1U << 3), 86 flag_opt_utrace = (1U << 4), 87 flag_in_valgrind = (1U << 5), 88 flag_opt_xmalloc = (1U << 6) 89 }; 90 static uint8_t malloc_slow_flags; 91 92 JEMALLOC_ALIGNED(CACHELINE) 93 const size_t pind2sz_tab[NPSIZES] = { 94 #define PSZ_yes(lg_grp, ndelta, lg_delta) \ 95 (((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta))), 96 #define PSZ_no(lg_grp, ndelta, lg_delta) 97 #define SC(index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup) \ 98 PSZ_##psz(lg_grp, ndelta, lg_delta) 99 SIZE_CLASSES 100 #undef PSZ_yes 101 #undef PSZ_no 102 #undef SC 103 }; 104 105 JEMALLOC_ALIGNED(CACHELINE) 106 const size_t index2size_tab[NSIZES] = { 107 #define SC(index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup) \ 108 ((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)), 109 SIZE_CLASSES 110 #undef SC 111 }; 112 113 JEMALLOC_ALIGNED(CACHELINE) 114 const uint8_t size2index_tab[] = { 115 #if LG_TINY_MIN == 0 116 #warning "Dangerous LG_TINY_MIN" 117 #define S2B_0(i) i, 118 #elif LG_TINY_MIN == 1 119 #warning "Dangerous LG_TINY_MIN" 120 #define S2B_1(i) i, 121 #elif LG_TINY_MIN == 2 122 #warning "Dangerous LG_TINY_MIN" 123 #define S2B_2(i) i, 124 #elif LG_TINY_MIN == 3 125 #define S2B_3(i) i, 126 #elif LG_TINY_MIN == 4 127 #define S2B_4(i) i, 128 #elif LG_TINY_MIN == 5 129 #define S2B_5(i) i, 130 #elif LG_TINY_MIN == 6 131 #define S2B_6(i) i, 132 #elif LG_TINY_MIN == 7 133 #define S2B_7(i) i, 134 #elif LG_TINY_MIN == 8 135 #define S2B_8(i) i, 136 #elif LG_TINY_MIN == 9 137 #define S2B_9(i) i, 138 #elif LG_TINY_MIN == 10 139 #define S2B_10(i) i, 140 #elif LG_TINY_MIN == 11 141 #define S2B_11(i) i, 142 #else 143 #error "Unsupported LG_TINY_MIN" 144 #endif 145 #if LG_TINY_MIN < 1 146 #define S2B_1(i) S2B_0(i) S2B_0(i) 147 #endif 148 #if LG_TINY_MIN < 2 149 #define S2B_2(i) S2B_1(i) S2B_1(i) 150 #endif 151 #if LG_TINY_MIN < 3 152 #define S2B_3(i) S2B_2(i) S2B_2(i) 153 #endif 154 #if LG_TINY_MIN < 4 155 #define S2B_4(i) S2B_3(i) S2B_3(i) 156 #endif 157 #if LG_TINY_MIN < 5 158 #define S2B_5(i) S2B_4(i) S2B_4(i) 159 #endif 160 #if LG_TINY_MIN < 6 161 #define S2B_6(i) S2B_5(i) S2B_5(i) 162 #endif 163 #if LG_TINY_MIN < 7 164 #define S2B_7(i) S2B_6(i) S2B_6(i) 165 #endif 166 #if LG_TINY_MIN < 8 167 #define S2B_8(i) S2B_7(i) S2B_7(i) 168 #endif 169 #if LG_TINY_MIN < 9 170 #define S2B_9(i) S2B_8(i) S2B_8(i) 171 #endif 172 #if LG_TINY_MIN < 10 173 #define S2B_10(i) S2B_9(i) S2B_9(i) 174 #endif 175 #if LG_TINY_MIN < 11 176 #define S2B_11(i) S2B_10(i) S2B_10(i) 177 #endif 178 #define S2B_no(i) 179 #define SC(index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup) \ 180 S2B_##lg_delta_lookup(index) 181 SIZE_CLASSES 182 #undef S2B_3 183 #undef S2B_4 184 #undef S2B_5 185 #undef S2B_6 186 #undef S2B_7 187 #undef S2B_8 188 #undef S2B_9 189 #undef S2B_10 190 #undef S2B_11 191 #undef S2B_no 192 #undef SC 193 }; 194 195 #ifdef JEMALLOC_THREADED_INIT 196 /* Used to let the initializing thread recursively allocate. */ 197 # define NO_INITIALIZER ((unsigned long)0) 198 # define INITIALIZER pthread_self() 199 # define IS_INITIALIZER (malloc_initializer == pthread_self()) 200 static pthread_t malloc_initializer = NO_INITIALIZER; 201 #else 202 # define NO_INITIALIZER false 203 # define INITIALIZER true 204 # define IS_INITIALIZER malloc_initializer 205 static bool malloc_initializer = NO_INITIALIZER; 206 #endif 207 208 /* Used to avoid initialization races. */ 209 #ifdef _WIN32 210 #if _WIN32_WINNT >= 0x0600 211 static malloc_mutex_t init_lock = SRWLOCK_INIT; 212 #else 213 static malloc_mutex_t init_lock; 214 static bool init_lock_initialized = false; 215 216 JEMALLOC_ATTR(constructor) 217 static void WINAPI 218 _init_init_lock(void) 219 { 220 221 /* If another constructor in the same binary is using mallctl to 222 * e.g. setup chunk hooks, it may end up running before this one, 223 * and malloc_init_hard will crash trying to lock the uninitialized 224 * lock. So we force an initialization of the lock in 225 * malloc_init_hard as well. We don't try to care about atomicity 226 * of the accessed to the init_lock_initialized boolean, since it 227 * really only matters early in the process creation, before any 228 * separate thread normally starts doing anything. */ 229 if (!init_lock_initialized) 230 malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT); 231 init_lock_initialized = true; 232 } 233 234 #ifdef _MSC_VER 235 # pragma section(".CRT$XCU", read) 236 JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used) 237 static const void (WINAPI *init_init_lock)(void) = _init_init_lock; 238 #endif 239 #endif 240 #else 241 static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; 242 #endif 243 244 typedef struct { 245 void *p; /* Input pointer (as in realloc(p, s)). */ 246 size_t s; /* Request size. */ 247 void *r; /* Result pointer. */ 248 } malloc_utrace_t; 249 250 #ifdef JEMALLOC_UTRACE 251 # define UTRACE(a, b, c) do { \ 252 if (unlikely(opt_utrace)) { \ 253 int utrace_serrno = errno; \ 254 malloc_utrace_t ut; \ 255 ut.p = (a); \ 256 ut.s = (b); \ 257 ut.r = (c); \ 258 utrace(&ut, sizeof(ut)); \ 259 errno = utrace_serrno; \ 260 } \ 261 } while (0) 262 #else 263 # define UTRACE(a, b, c) 264 #endif 265 266 /******************************************************************************/ 267 /* 268 * Function prototypes for static functions that are referenced prior to 269 * definition. 270 */ 271 272 static bool malloc_init_hard_a0(void); 273 static bool malloc_init_hard(void); 274 275 /******************************************************************************/ 276 /* 277 * Begin miscellaneous support functions. 278 */ 279 280 JEMALLOC_ALWAYS_INLINE_C bool 281 malloc_initialized(void) 282 { 283 284 return (malloc_init_state == malloc_init_initialized); 285 } 286 287 JEMALLOC_ALWAYS_INLINE_C void 288 malloc_thread_init(void) 289 { 290 291 /* 292 * TSD initialization can't be safely done as a side effect of 293 * deallocation, because it is possible for a thread to do nothing but 294 * deallocate its TLS data via free(), in which case writing to TLS 295 * would cause write-after-free memory corruption. The quarantine 296 * facility *only* gets used as a side effect of deallocation, so make 297 * a best effort attempt at initializing its TSD by hooking all 298 * allocation events. 299 */ 300 if (config_fill && unlikely(opt_quarantine)) 301 quarantine_alloc_hook(); 302 } 303 304 JEMALLOC_ALWAYS_INLINE_C bool 305 malloc_init_a0(void) 306 { 307 308 if (unlikely(malloc_init_state == malloc_init_uninitialized)) 309 return (malloc_init_hard_a0()); 310 return (false); 311 } 312 313 JEMALLOC_ALWAYS_INLINE_C bool 314 malloc_init(void) 315 { 316 317 if (unlikely(!malloc_initialized()) && malloc_init_hard()) 318 return (true); 319 malloc_thread_init(); 320 321 return (false); 322 } 323 324 /* 325 * The a0*() functions are used instead of i{d,}alloc() in situations that 326 * cannot tolerate TLS variable access. 327 */ 328 329 static void * 330 a0ialloc(size_t size, bool zero, bool is_metadata) 331 { 332 333 if (unlikely(malloc_init_a0())) 334 return (NULL); 335 336 return (iallocztm(TSDN_NULL, size, size2index(size), zero, NULL, 337 is_metadata, arena_get(TSDN_NULL, 0, true), true)); 338 } 339 340 static void 341 a0idalloc(void *ptr, bool is_metadata) 342 { 343 344 idalloctm(TSDN_NULL, ptr, false, is_metadata, true); 345 } 346 347 arena_t * 348 a0get(void) 349 { 350 351 return (a0); 352 } 353 354 void * 355 a0malloc(size_t size) 356 { 357 358 return (a0ialloc(size, false, true)); 359 } 360 361 void 362 a0dalloc(void *ptr) 363 { 364 365 a0idalloc(ptr, true); 366 } 367 368 /* 369 * FreeBSD's libc uses the bootstrap_*() functions in bootstrap-senstive 370 * situations that cannot tolerate TLS variable access (TLS allocation and very 371 * early internal data structure initialization). 372 */ 373 374 void * 375 bootstrap_malloc(size_t size) 376 { 377 378 if (unlikely(size == 0)) 379 size = 1; 380 381 return (a0ialloc(size, false, false)); 382 } 383 384 void * 385 bootstrap_calloc(size_t num, size_t size) 386 { 387 size_t num_size; 388 389 num_size = num * size; 390 if (unlikely(num_size == 0)) { 391 assert(num == 0 || size == 0); 392 num_size = 1; 393 } 394 395 return (a0ialloc(num_size, true, false)); 396 } 397 398 void 399 bootstrap_free(void *ptr) 400 { 401 402 if (unlikely(ptr == NULL)) 403 return; 404 405 a0idalloc(ptr, false); 406 } 407 408 static void 409 arena_set(unsigned ind, arena_t *arena) 410 { 411 412 atomic_write_p((void **)&arenas[ind], arena); 413 } 414 415 static void 416 narenas_total_set(unsigned narenas) 417 { 418 419 atomic_write_u(&narenas_total, narenas); 420 } 421 422 static void 423 narenas_total_inc(void) 424 { 425 426 atomic_add_u(&narenas_total, 1); 427 } 428 429 unsigned 430 narenas_total_get(void) 431 { 432 433 return (atomic_read_u(&narenas_total)); 434 } 435 436 /* Create a new arena and insert it into the arenas array at index ind. */ 437 static arena_t * 438 arena_init_locked(tsdn_t *tsdn, unsigned ind) 439 { 440 arena_t *arena; 441 442 assert(ind <= narenas_total_get()); 443 if (ind > MALLOCX_ARENA_MAX) 444 return (NULL); 445 if (ind == narenas_total_get()) 446 narenas_total_inc(); 447 448 /* 449 * Another thread may have already initialized arenas[ind] if it's an 450 * auto arena. 451 */ 452 arena = arena_get(tsdn, ind, false); 453 if (arena != NULL) { 454 assert(ind < narenas_auto); 455 return (arena); 456 } 457 458 /* Actually initialize the arena. */ 459 arena = arena_new(tsdn, ind); 460 arena_set(ind, arena); 461 return (arena); 462 } 463 464 arena_t * 465 arena_init(tsdn_t *tsdn, unsigned ind) 466 { 467 arena_t *arena; 468 469 malloc_mutex_lock(tsdn, &arenas_lock); 470 arena = arena_init_locked(tsdn, ind); 471 malloc_mutex_unlock(tsdn, &arenas_lock); 472 return (arena); 473 } 474 475 static void 476 arena_bind(tsd_t *tsd, unsigned ind, bool internal) 477 { 478 arena_t *arena; 479 480 if (!tsd_nominal(tsd)) 481 return; 482 483 arena = arena_get(tsd_tsdn(tsd), ind, false); 484 arena_nthreads_inc(arena, internal); 485 486 if (internal) 487 tsd_iarena_set(tsd, arena); 488 else 489 tsd_arena_set(tsd, arena); 490 } 491 492 void 493 arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind) 494 { 495 arena_t *oldarena, *newarena; 496 497 oldarena = arena_get(tsd_tsdn(tsd), oldind, false); 498 newarena = arena_get(tsd_tsdn(tsd), newind, false); 499 arena_nthreads_dec(oldarena, false); 500 arena_nthreads_inc(newarena, false); 501 tsd_arena_set(tsd, newarena); 502 } 503 504 static void 505 arena_unbind(tsd_t *tsd, unsigned ind, bool internal) 506 { 507 arena_t *arena; 508 509 arena = arena_get(tsd_tsdn(tsd), ind, false); 510 arena_nthreads_dec(arena, internal); 511 if (internal) 512 tsd_iarena_set(tsd, NULL); 513 else 514 tsd_arena_set(tsd, NULL); 515 } 516 517 arena_tdata_t * 518 arena_tdata_get_hard(tsd_t *tsd, unsigned ind) 519 { 520 arena_tdata_t *tdata, *arenas_tdata_old; 521 arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd); 522 unsigned narenas_tdata_old, i; 523 unsigned narenas_tdata = tsd_narenas_tdata_get(tsd); 524 unsigned narenas_actual = narenas_total_get(); 525 526 /* 527 * Dissociate old tdata array (and set up for deallocation upon return) 528 * if it's too small. 529 */ 530 if (arenas_tdata != NULL && narenas_tdata < narenas_actual) { 531 arenas_tdata_old = arenas_tdata; 532 narenas_tdata_old = narenas_tdata; 533 arenas_tdata = NULL; 534 narenas_tdata = 0; 535 tsd_arenas_tdata_set(tsd, arenas_tdata); 536 tsd_narenas_tdata_set(tsd, narenas_tdata); 537 } else { 538 arenas_tdata_old = NULL; 539 narenas_tdata_old = 0; 540 } 541 542 /* Allocate tdata array if it's missing. */ 543 if (arenas_tdata == NULL) { 544 bool *arenas_tdata_bypassp = tsd_arenas_tdata_bypassp_get(tsd); 545 narenas_tdata = (ind < narenas_actual) ? narenas_actual : ind+1; 546 547 if (tsd_nominal(tsd) && !*arenas_tdata_bypassp) { 548 *arenas_tdata_bypassp = true; 549 arenas_tdata = (arena_tdata_t *)a0malloc( 550 sizeof(arena_tdata_t) * narenas_tdata); 551 *arenas_tdata_bypassp = false; 552 } 553 if (arenas_tdata == NULL) { 554 tdata = NULL; 555 goto label_return; 556 } 557 assert(tsd_nominal(tsd) && !*arenas_tdata_bypassp); 558 tsd_arenas_tdata_set(tsd, arenas_tdata); 559 tsd_narenas_tdata_set(tsd, narenas_tdata); 560 } 561 562 /* 563 * Copy to tdata array. It's possible that the actual number of arenas 564 * has increased since narenas_total_get() was called above, but that 565 * causes no correctness issues unless two threads concurrently execute 566 * the arenas.extend mallctl, which we trust mallctl synchronization to 567 * prevent. 568 */ 569 570 /* Copy/initialize tickers. */ 571 for (i = 0; i < narenas_actual; i++) { 572 if (i < narenas_tdata_old) { 573 ticker_copy(&arenas_tdata[i].decay_ticker, 574 &arenas_tdata_old[i].decay_ticker); 575 } else { 576 ticker_init(&arenas_tdata[i].decay_ticker, 577 DECAY_NTICKS_PER_UPDATE); 578 } 579 } 580 if (narenas_tdata > narenas_actual) { 581 memset(&arenas_tdata[narenas_actual], 0, sizeof(arena_tdata_t) 582 * (narenas_tdata - narenas_actual)); 583 } 584 585 /* Read the refreshed tdata array. */ 586 tdata = &arenas_tdata[ind]; 587 label_return: 588 if (arenas_tdata_old != NULL) 589 a0dalloc(arenas_tdata_old); 590 return (tdata); 591 } 592 593 /* Slow path, called only by arena_choose(). */ 594 arena_t * 595 arena_choose_hard(tsd_t *tsd, bool internal) 596 { 597 arena_t *ret JEMALLOC_CC_SILENCE_INIT(NULL); 598 599 if (narenas_auto > 1) { 600 unsigned i, j, choose[2], first_null; 601 602 /* 603 * Determine binding for both non-internal and internal 604 * allocation. 605 * 606 * choose[0]: For application allocation. 607 * choose[1]: For internal metadata allocation. 608 */ 609 610 for (j = 0; j < 2; j++) 611 choose[j] = 0; 612 613 first_null = narenas_auto; 614 malloc_mutex_lock(tsd_tsdn(tsd), &arenas_lock); 615 assert(arena_get(tsd_tsdn(tsd), 0, false) != NULL); 616 for (i = 1; i < narenas_auto; i++) { 617 if (arena_get(tsd_tsdn(tsd), i, false) != NULL) { 618 /* 619 * Choose the first arena that has the lowest 620 * number of threads assigned to it. 621 */ 622 for (j = 0; j < 2; j++) { 623 if (arena_nthreads_get(arena_get( 624 tsd_tsdn(tsd), i, false), !!j) < 625 arena_nthreads_get(arena_get( 626 tsd_tsdn(tsd), choose[j], false), 627 !!j)) 628 choose[j] = i; 629 } 630 } else if (first_null == narenas_auto) { 631 /* 632 * Record the index of the first uninitialized 633 * arena, in case all extant arenas are in use. 634 * 635 * NB: It is possible for there to be 636 * discontinuities in terms of initialized 637 * versus uninitialized arenas, due to the 638 * "thread.arena" mallctl. 639 */ 640 first_null = i; 641 } 642 } 643 644 for (j = 0; j < 2; j++) { 645 if (arena_nthreads_get(arena_get(tsd_tsdn(tsd), 646 choose[j], false), !!j) == 0 || first_null == 647 narenas_auto) { 648 /* 649 * Use an unloaded arena, or the least loaded 650 * arena if all arenas are already initialized. 651 */ 652 if (!!j == internal) { 653 ret = arena_get(tsd_tsdn(tsd), 654 choose[j], false); 655 } 656 } else { 657 arena_t *arena; 658 659 /* Initialize a new arena. */ 660 choose[j] = first_null; 661 arena = arena_init_locked(tsd_tsdn(tsd), 662 choose[j]); 663 if (arena == NULL) { 664 malloc_mutex_unlock(tsd_tsdn(tsd), 665 &arenas_lock); 666 return (NULL); 667 } 668 if (!!j == internal) 669 ret = arena; 670 } 671 arena_bind(tsd, choose[j], !!j); 672 } 673 malloc_mutex_unlock(tsd_tsdn(tsd), &arenas_lock); 674 } else { 675 ret = arena_get(tsd_tsdn(tsd), 0, false); 676 arena_bind(tsd, 0, false); 677 arena_bind(tsd, 0, true); 678 } 679 680 return (ret); 681 } 682 683 void 684 thread_allocated_cleanup(tsd_t *tsd) 685 { 686 687 /* Do nothing. */ 688 } 689 690 void 691 thread_deallocated_cleanup(tsd_t *tsd) 692 { 693 694 /* Do nothing. */ 695 } 696 697 void 698 iarena_cleanup(tsd_t *tsd) 699 { 700 arena_t *iarena; 701 702 iarena = tsd_iarena_get(tsd); 703 if (iarena != NULL) 704 arena_unbind(tsd, iarena->ind, true); 705 } 706 707 void 708 arena_cleanup(tsd_t *tsd) 709 { 710 arena_t *arena; 711 712 arena = tsd_arena_get(tsd); 713 if (arena != NULL) 714 arena_unbind(tsd, arena->ind, false); 715 } 716 717 void 718 arenas_tdata_cleanup(tsd_t *tsd) 719 { 720 arena_tdata_t *arenas_tdata; 721 722 /* Prevent tsd->arenas_tdata from being (re)created. */ 723 *tsd_arenas_tdata_bypassp_get(tsd) = true; 724 725 arenas_tdata = tsd_arenas_tdata_get(tsd); 726 if (arenas_tdata != NULL) { 727 tsd_arenas_tdata_set(tsd, NULL); 728 a0dalloc(arenas_tdata); 729 } 730 } 731 732 void 733 narenas_tdata_cleanup(tsd_t *tsd) 734 { 735 736 /* Do nothing. */ 737 } 738 739 void 740 arenas_tdata_bypass_cleanup(tsd_t *tsd) 741 { 742 743 /* Do nothing. */ 744 } 745 746 static void 747 stats_print_atexit(void) 748 { 749 750 if (config_tcache && config_stats) { 751 tsdn_t *tsdn; 752 unsigned narenas, i; 753 754 tsdn = tsdn_fetch(); 755 756 /* 757 * Merge stats from extant threads. This is racy, since 758 * individual threads do not lock when recording tcache stats 759 * events. As a consequence, the final stats may be slightly 760 * out of date by the time they are reported, if other threads 761 * continue to allocate. 762 */ 763 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 764 arena_t *arena = arena_get(tsdn, i, false); 765 if (arena != NULL) { 766 tcache_t *tcache; 767 768 /* 769 * tcache_stats_merge() locks bins, so if any 770 * code is introduced that acquires both arena 771 * and bin locks in the opposite order, 772 * deadlocks may result. 773 */ 774 malloc_mutex_lock(tsdn, &arena->lock); 775 ql_foreach(tcache, &arena->tcache_ql, link) { 776 tcache_stats_merge(tsdn, tcache, arena); 777 } 778 malloc_mutex_unlock(tsdn, &arena->lock); 779 } 780 } 781 } 782 je_malloc_stats_print(NULL, NULL, NULL); 783 } 784 785 /* 786 * End miscellaneous support functions. 787 */ 788 /******************************************************************************/ 789 /* 790 * Begin initialization functions. 791 */ 792 793 #ifndef JEMALLOC_HAVE_SECURE_GETENV 794 static char * 795 secure_getenv(const char *name) 796 { 797 798 # ifdef JEMALLOC_HAVE_ISSETUGID 799 if (issetugid() != 0) 800 return (NULL); 801 # endif 802 return (getenv(name)); 803 } 804 #endif 805 806 static unsigned 807 malloc_ncpus(void) 808 { 809 long result; 810 811 #ifdef _WIN32 812 SYSTEM_INFO si; 813 GetSystemInfo(&si); 814 result = si.dwNumberOfProcessors; 815 #elif defined(JEMALLOC_GLIBC_MALLOC_HOOK) && defined(CPU_COUNT) 816 /* 817 * glibc >= 2.6 has the CPU_COUNT macro. 818 * 819 * glibc's sysconf() uses isspace(). glibc allocates for the first time 820 * *before* setting up the isspace tables. Therefore we need a 821 * different method to get the number of CPUs. 822 */ 823 { 824 cpu_set_t set; 825 826 pthread_getaffinity_np(pthread_self(), sizeof(set), &set); 827 result = CPU_COUNT(&set); 828 } 829 #else 830 result = sysconf(_SC_NPROCESSORS_ONLN); 831 #endif 832 return ((result == -1) ? 1 : (unsigned)result); 833 } 834 835 static bool 836 malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, 837 char const **v_p, size_t *vlen_p) 838 { 839 bool accept; 840 const char *opts = *opts_p; 841 842 *k_p = opts; 843 844 for (accept = false; !accept;) { 845 switch (*opts) { 846 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': 847 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': 848 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': 849 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': 850 case 'Y': case 'Z': 851 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': 852 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': 853 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 854 case 's': case 't': case 'u': case 'v': case 'w': case 'x': 855 case 'y': case 'z': 856 case '0': case '1': case '2': case '3': case '4': case '5': 857 case '6': case '7': case '8': case '9': 858 case '_': 859 opts++; 860 break; 861 case ':': 862 opts++; 863 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; 864 *v_p = opts; 865 accept = true; 866 break; 867 case '\0': 868 if (opts != *opts_p) { 869 malloc_write("<jemalloc>: Conf string ends " 870 "with key\n"); 871 } 872 return (true); 873 default: 874 malloc_write("<jemalloc>: Malformed conf string\n"); 875 return (true); 876 } 877 } 878 879 for (accept = false; !accept;) { 880 switch (*opts) { 881 case ',': 882 opts++; 883 /* 884 * Look ahead one character here, because the next time 885 * this function is called, it will assume that end of 886 * input has been cleanly reached if no input remains, 887 * but we have optimistically already consumed the 888 * comma if one exists. 889 */ 890 if (*opts == '\0') { 891 malloc_write("<jemalloc>: Conf string ends " 892 "with comma\n"); 893 } 894 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; 895 accept = true; 896 break; 897 case '\0': 898 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; 899 accept = true; 900 break; 901 default: 902 opts++; 903 break; 904 } 905 } 906 907 *opts_p = opts; 908 return (false); 909 } 910 911 static void 912 malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, 913 size_t vlen) 914 { 915 916 malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k, 917 (int)vlen, v); 918 } 919 920 static void 921 malloc_slow_flag_init(void) 922 { 923 /* 924 * Combine the runtime options into malloc_slow for fast path. Called 925 * after processing all the options. 926 */ 927 malloc_slow_flags |= (opt_junk_alloc ? flag_opt_junk_alloc : 0) 928 | (opt_junk_free ? flag_opt_junk_free : 0) 929 | (opt_quarantine ? flag_opt_quarantine : 0) 930 | (opt_zero ? flag_opt_zero : 0) 931 | (opt_utrace ? flag_opt_utrace : 0) 932 | (opt_xmalloc ? flag_opt_xmalloc : 0); 933 934 if (config_valgrind) 935 malloc_slow_flags |= (in_valgrind ? flag_in_valgrind : 0); 936 937 malloc_slow = (malloc_slow_flags != 0); 938 } 939 940 static void 941 malloc_conf_init(void) 942 { 943 unsigned i; 944 char buf[PATH_MAX + 1]; 945 const char *opts, *k, *v; 946 size_t klen, vlen; 947 948 /* 949 * Automatically configure valgrind before processing options. The 950 * valgrind option remains in jemalloc 3.x for compatibility reasons. 951 */ 952 if (config_valgrind) { 953 in_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false; 954 if (config_fill && unlikely(in_valgrind)) { 955 opt_junk = "false"; 956 opt_junk_alloc = false; 957 opt_junk_free = false; 958 assert(!opt_zero); 959 opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; 960 opt_redzone = true; 961 } 962 if (config_tcache && unlikely(in_valgrind)) 963 opt_tcache = false; 964 } 965 966 #if defined(__ANDROID__) 967 for (i = 0; i < 2; i++) { 968 #else 969 for (i = 0; i < 4; i++) { 970 #endif 971 /* Get runtime configuration. */ 972 switch (i) { 973 case 0: 974 opts = config_malloc_conf; 975 break; 976 case 1: 977 if (je_malloc_conf != NULL) { 978 /* 979 * Use options that were compiled into the 980 * program. 981 */ 982 opts = je_malloc_conf; 983 } else { 984 /* No configuration specified. */ 985 buf[0] = '\0'; 986 opts = buf; 987 } 988 break; 989 case 2: { 990 ssize_t linklen = 0; 991 #ifndef _WIN32 992 int saved_errno = errno; 993 const char *linkname = 994 # ifdef JEMALLOC_PREFIX 995 "/etc/"JEMALLOC_PREFIX"malloc.conf" 996 # else 997 "/etc/malloc.conf" 998 # endif 999 ; 1000 1001 /* 1002 * Try to use the contents of the "/etc/malloc.conf" 1003 * symbolic link's name. 1004 */ 1005 linklen = readlink(linkname, buf, sizeof(buf) - 1); 1006 if (linklen == -1) { 1007 /* No configuration specified. */ 1008 linklen = 0; 1009 /* Restore errno. */ 1010 set_errno(saved_errno); 1011 } 1012 #endif 1013 buf[linklen] = '\0'; 1014 opts = buf; 1015 break; 1016 } case 3: { 1017 const char *envname = 1018 #ifdef JEMALLOC_PREFIX 1019 JEMALLOC_CPREFIX"MALLOC_CONF" 1020 #else 1021 "MALLOC_CONF" 1022 #endif 1023 ; 1024 1025 if ((opts = secure_getenv(envname)) != NULL) { 1026 /* 1027 * Do nothing; opts is already initialized to 1028 * the value of the MALLOC_CONF environment 1029 * variable. 1030 */ 1031 } else { 1032 /* No configuration specified. */ 1033 buf[0] = '\0'; 1034 opts = buf; 1035 } 1036 break; 1037 } default: 1038 not_reached(); 1039 buf[0] = '\0'; 1040 opts = buf; 1041 } 1042 1043 while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v, 1044 &vlen)) { 1045 #define CONF_MATCH(n) \ 1046 (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0) 1047 #define CONF_MATCH_VALUE(n) \ 1048 (sizeof(n)-1 == vlen && strncmp(n, v, vlen) == 0) 1049 #define CONF_HANDLE_BOOL(o, n, cont) \ 1050 if (CONF_MATCH(n)) { \ 1051 if (CONF_MATCH_VALUE("true")) \ 1052 o = true; \ 1053 else if (CONF_MATCH_VALUE("false")) \ 1054 o = false; \ 1055 else { \ 1056 malloc_conf_error( \ 1057 "Invalid conf value", \ 1058 k, klen, v, vlen); \ 1059 } \ 1060 if (cont) \ 1061 continue; \ 1062 } 1063 #define CONF_MIN_no(um, min) false 1064 #define CONF_MIN_yes(um, min) ((um) < (min)) 1065 #define CONF_MAX_no(um, max) false 1066 #define CONF_MAX_yes(um, max) ((um) > (max)) 1067 #define CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip) \ 1068 if (CONF_MATCH(n)) { \ 1069 uintmax_t um; \ 1070 char *end; \ 1071 \ 1072 set_errno(0); \ 1073 um = malloc_strtoumax(v, &end, 0); \ 1074 if (get_errno() != 0 || (uintptr_t)end -\ 1075 (uintptr_t)v != vlen) { \ 1076 malloc_conf_error( \ 1077 "Invalid conf value", \ 1078 k, klen, v, vlen); \ 1079 } else if (clip) { \ 1080 if (CONF_MIN_##check_min(um, \ 1081 (min))) \ 1082 o = (t)(min); \ 1083 else if (CONF_MAX_##check_max( \ 1084 um, (max))) \ 1085 o = (t)(max); \ 1086 else \ 1087 o = (t)um; \ 1088 } else { \ 1089 if (CONF_MIN_##check_min(um, \ 1090 (min)) || \ 1091 CONF_MAX_##check_max(um, \ 1092 (max))) { \ 1093 malloc_conf_error( \ 1094 "Out-of-range " \ 1095 "conf value", \ 1096 k, klen, v, vlen); \ 1097 } else \ 1098 o = (t)um; \ 1099 } \ 1100 continue; \ 1101 } 1102 #define CONF_HANDLE_UNSIGNED(o, n, min, max, check_min, check_max, \ 1103 clip) \ 1104 CONF_HANDLE_T_U(unsigned, o, n, min, max, \ 1105 check_min, check_max, clip) 1106 #define CONF_HANDLE_SIZE_T(o, n, min, max, check_min, check_max, clip) \ 1107 CONF_HANDLE_T_U(size_t, o, n, min, max, \ 1108 check_min, check_max, clip) 1109 #define CONF_HANDLE_SSIZE_T(o, n, min, max) \ 1110 if (CONF_MATCH(n)) { \ 1111 long l; \ 1112 char *end; \ 1113 \ 1114 set_errno(0); \ 1115 l = strtol(v, &end, 0); \ 1116 if (get_errno() != 0 || (uintptr_t)end -\ 1117 (uintptr_t)v != vlen) { \ 1118 malloc_conf_error( \ 1119 "Invalid conf value", \ 1120 k, klen, v, vlen); \ 1121 } else if (l < (ssize_t)(min) || l > \ 1122 (ssize_t)(max)) { \ 1123 malloc_conf_error( \ 1124 "Out-of-range conf value", \ 1125 k, klen, v, vlen); \ 1126 } else \ 1127 o = l; \ 1128 continue; \ 1129 } 1130 #define CONF_HANDLE_CHAR_P(o, n, d) \ 1131 if (CONF_MATCH(n)) { \ 1132 size_t cpylen = (vlen <= \ 1133 sizeof(o)-1) ? vlen : \ 1134 sizeof(o)-1; \ 1135 strncpy(o, v, cpylen); \ 1136 o[cpylen] = '\0'; \ 1137 continue; \ 1138 } 1139 1140 CONF_HANDLE_BOOL(opt_abort, "abort", true) 1141 /* 1142 * Chunks always require at least one header page, 1143 * as many as 2^(LG_SIZE_CLASS_GROUP+1) data pages, and 1144 * possibly an additional page in the presence of 1145 * redzones. In order to simplify options processing, 1146 * use a conservative bound that accommodates all these 1147 * constraints. 1148 */ 1149 CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + 1150 LG_SIZE_CLASS_GROUP + (config_fill ? 2 : 1), 1151 (sizeof(size_t) << 3) - 1, yes, yes, true) 1152 if (strncmp("dss", k, klen) == 0) { 1153 int i; 1154 bool match = false; 1155 for (i = 0; i < dss_prec_limit; i++) { 1156 if (strncmp(dss_prec_names[i], v, vlen) 1157 == 0) { 1158 if (chunk_dss_prec_set(i)) { 1159 malloc_conf_error( 1160 "Error setting dss", 1161 k, klen, v, vlen); 1162 } else { 1163 opt_dss = 1164 dss_prec_names[i]; 1165 match = true; 1166 break; 1167 } 1168 } 1169 } 1170 if (!match) { 1171 malloc_conf_error("Invalid conf value", 1172 k, klen, v, vlen); 1173 } 1174 continue; 1175 } 1176 CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1, 1177 UINT_MAX, yes, no, false) 1178 if (strncmp("purge", k, klen) == 0) { 1179 int i; 1180 bool match = false; 1181 for (i = 0; i < purge_mode_limit; i++) { 1182 if (strncmp(purge_mode_names[i], v, 1183 vlen) == 0) { 1184 opt_purge = (purge_mode_t)i; 1185 match = true; 1186 break; 1187 } 1188 } 1189 if (!match) { 1190 malloc_conf_error("Invalid conf value", 1191 k, klen, v, vlen); 1192 } 1193 continue; 1194 } 1195 CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult", 1196 -1, (sizeof(size_t) << 3) - 1) 1197 CONF_HANDLE_SSIZE_T(opt_decay_time, "decay_time", -1, 1198 NSTIME_SEC_MAX); 1199 CONF_HANDLE_BOOL(opt_stats_print, "stats_print", true) 1200 if (config_fill) { 1201 if (CONF_MATCH("junk")) { 1202 if (CONF_MATCH_VALUE("true")) { 1203 if (config_valgrind && 1204 unlikely(in_valgrind)) { 1205 malloc_conf_error( 1206 "Deallocation-time " 1207 "junk filling cannot " 1208 "be enabled while " 1209 "running inside " 1210 "Valgrind", k, klen, v, 1211 vlen); 1212 } else { 1213 opt_junk = "true"; 1214 opt_junk_alloc = true; 1215 opt_junk_free = true; 1216 } 1217 } else if (CONF_MATCH_VALUE("false")) { 1218 opt_junk = "false"; 1219 opt_junk_alloc = opt_junk_free = 1220 false; 1221 } else if (CONF_MATCH_VALUE("alloc")) { 1222 opt_junk = "alloc"; 1223 opt_junk_alloc = true; 1224 opt_junk_free = false; 1225 } else if (CONF_MATCH_VALUE("free")) { 1226 if (config_valgrind && 1227 unlikely(in_valgrind)) { 1228 malloc_conf_error( 1229 "Deallocation-time " 1230 "junk filling cannot " 1231 "be enabled while " 1232 "running inside " 1233 "Valgrind", k, klen, v, 1234 vlen); 1235 } else { 1236 opt_junk = "free"; 1237 opt_junk_alloc = false; 1238 opt_junk_free = true; 1239 } 1240 } else { 1241 malloc_conf_error( 1242 "Invalid conf value", k, 1243 klen, v, vlen); 1244 } 1245 continue; 1246 } 1247 CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine", 1248 0, SIZE_T_MAX, no, no, false) 1249 CONF_HANDLE_BOOL(opt_redzone, "redzone", true) 1250 CONF_HANDLE_BOOL(opt_zero, "zero", true) 1251 } 1252 if (config_utrace) { 1253 CONF_HANDLE_BOOL(opt_utrace, "utrace", true) 1254 } 1255 if (config_xmalloc) { 1256 CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc", true) 1257 } 1258 if (config_tcache) { 1259 CONF_HANDLE_BOOL(opt_tcache, "tcache", 1260 !config_valgrind || !in_valgrind) 1261 if (CONF_MATCH("tcache")) { 1262 assert(config_valgrind && in_valgrind); 1263 if (opt_tcache) { 1264 opt_tcache = false; 1265 malloc_conf_error( 1266 "tcache cannot be enabled " 1267 "while running inside Valgrind", 1268 k, klen, v, vlen); 1269 } 1270 continue; 1271 } 1272 CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, 1273 "lg_tcache_max", -1, 1274 (sizeof(size_t) << 3) - 1) 1275 } 1276 if (config_prof) { 1277 CONF_HANDLE_BOOL(opt_prof, "prof", true) 1278 CONF_HANDLE_CHAR_P(opt_prof_prefix, 1279 "prof_prefix", "jeprof") 1280 CONF_HANDLE_BOOL(opt_prof_active, "prof_active", 1281 true) 1282 CONF_HANDLE_BOOL(opt_prof_thread_active_init, 1283 "prof_thread_active_init", true) 1284 CONF_HANDLE_SIZE_T(opt_lg_prof_sample, 1285 "lg_prof_sample", 0, (sizeof(uint64_t) << 3) 1286 - 1, no, yes, true) 1287 CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum", 1288 true) 1289 CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, 1290 "lg_prof_interval", -1, 1291 (sizeof(uint64_t) << 3) - 1) 1292 CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump", 1293 true) 1294 CONF_HANDLE_BOOL(opt_prof_final, "prof_final", 1295 true) 1296 CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak", 1297 true) 1298 } 1299 malloc_conf_error("Invalid conf pair", k, klen, v, 1300 vlen); 1301 #undef CONF_MATCH 1302 #undef CONF_MATCH_VALUE 1303 #undef CONF_HANDLE_BOOL 1304 #undef CONF_MIN_no 1305 #undef CONF_MIN_yes 1306 #undef CONF_MAX_no 1307 #undef CONF_MAX_yes 1308 #undef CONF_HANDLE_T_U 1309 #undef CONF_HANDLE_UNSIGNED 1310 #undef CONF_HANDLE_SIZE_T 1311 #undef CONF_HANDLE_SSIZE_T 1312 #undef CONF_HANDLE_CHAR_P 1313 } 1314 } 1315 } 1316 1317 static bool 1318 malloc_init_hard_needed(void) 1319 { 1320 1321 if (malloc_initialized() || (IS_INITIALIZER && malloc_init_state == 1322 malloc_init_recursible)) { 1323 /* 1324 * Another thread initialized the allocator before this one 1325 * acquired init_lock, or this thread is the initializing 1326 * thread, and it is recursively allocating. 1327 */ 1328 return (false); 1329 } 1330 #ifdef JEMALLOC_THREADED_INIT 1331 if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) { 1332 spin_t spinner; 1333 1334 /* Busy-wait until the initializing thread completes. */ 1335 spin_init(&spinner); 1336 do { 1337 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1338 spin_adaptive(&spinner); 1339 malloc_mutex_lock(TSDN_NULL, &init_lock); 1340 } while (!malloc_initialized()); 1341 return (false); 1342 } 1343 #endif 1344 return (true); 1345 } 1346 1347 static bool 1348 malloc_init_hard_a0_locked() 1349 { 1350 1351 malloc_initializer = INITIALIZER; 1352 1353 if (config_prof) 1354 prof_boot0(); 1355 malloc_conf_init(); 1356 if (opt_stats_print) { 1357 /* Print statistics at exit. */ 1358 if (atexit(stats_print_atexit) != 0) { 1359 malloc_write("<jemalloc>: Error in atexit()\n"); 1360 if (opt_abort) 1361 abort(); 1362 } 1363 } 1364 pages_boot(); 1365 if (base_boot()) 1366 return (true); 1367 if (chunk_boot()) 1368 return (true); 1369 if (ctl_boot()) 1370 return (true); 1371 if (config_prof) 1372 prof_boot1(); 1373 arena_boot(); 1374 if (config_tcache && tcache_boot(TSDN_NULL)) 1375 return (true); 1376 if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS)) 1377 return (true); 1378 /* 1379 * Create enough scaffolding to allow recursive allocation in 1380 * malloc_ncpus(). 1381 */ 1382 narenas_auto = 1; 1383 narenas_total_set(narenas_auto); 1384 arenas = &a0; 1385 memset(arenas, 0, sizeof(arena_t *) * narenas_auto); 1386 /* 1387 * Initialize one arena here. The rest are lazily created in 1388 * arena_choose_hard(). 1389 */ 1390 if (arena_init(TSDN_NULL, 0) == NULL) 1391 return (true); 1392 1393 malloc_init_state = malloc_init_a0_initialized; 1394 1395 return (false); 1396 } 1397 1398 static bool 1399 malloc_init_hard_a0(void) 1400 { 1401 bool ret; 1402 1403 malloc_mutex_lock(TSDN_NULL, &init_lock); 1404 ret = malloc_init_hard_a0_locked(); 1405 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1406 return (ret); 1407 } 1408 1409 /* Initialize data structures which may trigger recursive allocation. */ 1410 static bool 1411 malloc_init_hard_recursible(void) 1412 { 1413 1414 malloc_init_state = malloc_init_recursible; 1415 1416 ncpus = malloc_ncpus(); 1417 1418 #if (defined(JEMALLOC_HAVE_PTHREAD_ATFORK) && !defined(JEMALLOC_MUTEX_INIT_CB) \ 1419 && !defined(JEMALLOC_ZONE) && !defined(_WIN32) && \ 1420 !defined(__native_client__)) 1421 /* LinuxThreads' pthread_atfork() allocates. */ 1422 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, 1423 jemalloc_postfork_child) != 0) { 1424 malloc_write("<jemalloc>: Error in pthread_atfork()\n"); 1425 if (opt_abort) 1426 abort(); 1427 return (true); 1428 } 1429 #endif 1430 1431 return (false); 1432 } 1433 1434 static bool 1435 malloc_init_hard_finish(tsdn_t *tsdn) 1436 { 1437 1438 if (malloc_mutex_boot()) 1439 return (true); 1440 1441 if (opt_narenas == 0) { 1442 /* 1443 * For SMP systems, create more than one arena per CPU by 1444 * default. 1445 */ 1446 if (ncpus > 1) 1447 opt_narenas = ncpus << 2; 1448 else 1449 opt_narenas = 1; 1450 } 1451 #if defined(ANDROID_MAX_ARENAS) 1452 /* Never create more than MAX_ARENAS arenas regardless of num_cpus. 1453 * Extra arenas use more PSS and are not very useful unless 1454 * lots of threads are allocing/freeing at the same time. 1455 */ 1456 if (opt_narenas > ANDROID_MAX_ARENAS) 1457 opt_narenas = ANDROID_MAX_ARENAS; 1458 #endif 1459 narenas_auto = opt_narenas; 1460 /* 1461 * Limit the number of arenas to the indexing range of MALLOCX_ARENA(). 1462 */ 1463 if (narenas_auto > MALLOCX_ARENA_MAX) { 1464 narenas_auto = MALLOCX_ARENA_MAX; 1465 malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n", 1466 narenas_auto); 1467 } 1468 narenas_total_set(narenas_auto); 1469 1470 /* Allocate and initialize arenas. */ 1471 arenas = (arena_t **)base_alloc(tsdn, sizeof(arena_t *) * 1472 (MALLOCX_ARENA_MAX+1)); 1473 if (arenas == NULL) 1474 return (true); 1475 /* Copy the pointer to the one arena that was already initialized. */ 1476 arena_set(0, a0); 1477 1478 malloc_init_state = malloc_init_initialized; 1479 malloc_slow_flag_init(); 1480 1481 return (false); 1482 } 1483 1484 static bool 1485 malloc_init_hard(void) 1486 { 1487 tsd_t *tsd; 1488 1489 #if defined(_WIN32) && _WIN32_WINNT < 0x0600 1490 _init_init_lock(); 1491 #endif 1492 malloc_mutex_lock(TSDN_NULL, &init_lock); 1493 if (!malloc_init_hard_needed()) { 1494 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1495 return (false); 1496 } 1497 1498 if (malloc_init_state != malloc_init_a0_initialized && 1499 malloc_init_hard_a0_locked()) { 1500 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1501 return (true); 1502 } 1503 1504 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1505 /* Recursive allocation relies on functional tsd. */ 1506 tsd = malloc_tsd_boot0(); 1507 if (tsd == NULL) 1508 return (true); 1509 if (malloc_init_hard_recursible()) 1510 return (true); 1511 malloc_mutex_lock(tsd_tsdn(tsd), &init_lock); 1512 1513 if (config_prof && prof_boot2(tsd)) { 1514 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock); 1515 return (true); 1516 } 1517 1518 if (malloc_init_hard_finish(tsd_tsdn(tsd))) { 1519 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock); 1520 return (true); 1521 } 1522 1523 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock); 1524 malloc_tsd_boot1(); 1525 return (false); 1526 } 1527 1528 /* 1529 * End initialization functions. 1530 */ 1531 /******************************************************************************/ 1532 /* 1533 * Begin malloc(3)-compatible functions. 1534 */ 1535 1536 static void * 1537 ialloc_prof_sample(tsd_t *tsd, size_t usize, szind_t ind, bool zero, 1538 prof_tctx_t *tctx, bool slow_path) 1539 { 1540 void *p; 1541 1542 if (tctx == NULL) 1543 return (NULL); 1544 if (usize <= SMALL_MAXCLASS) { 1545 szind_t ind_large = size2index(LARGE_MINCLASS); 1546 p = ialloc(tsd, LARGE_MINCLASS, ind_large, zero, slow_path); 1547 if (p == NULL) 1548 return (NULL); 1549 arena_prof_promoted(tsd_tsdn(tsd), p, usize); 1550 } else 1551 p = ialloc(tsd, usize, ind, zero, slow_path); 1552 1553 return (p); 1554 } 1555 1556 JEMALLOC_ALWAYS_INLINE_C void * 1557 ialloc_prof(tsd_t *tsd, size_t usize, szind_t ind, bool zero, bool slow_path) 1558 { 1559 void *p; 1560 prof_tctx_t *tctx; 1561 1562 tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true); 1563 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1564 p = ialloc_prof_sample(tsd, usize, ind, zero, tctx, slow_path); 1565 else 1566 p = ialloc(tsd, usize, ind, zero, slow_path); 1567 if (unlikely(p == NULL)) { 1568 prof_alloc_rollback(tsd, tctx, true); 1569 return (NULL); 1570 } 1571 prof_malloc(tsd_tsdn(tsd), p, usize, tctx); 1572 1573 return (p); 1574 } 1575 1576 /* 1577 * ialloc_body() is inlined so that fast and slow paths are generated separately 1578 * with statically known slow_path. 1579 * 1580 * This function guarantees that *tsdn is non-NULL on success. 1581 */ 1582 JEMALLOC_ALWAYS_INLINE_C void * 1583 ialloc_body(size_t size, bool zero, tsdn_t **tsdn, size_t *usize, 1584 bool slow_path) 1585 { 1586 tsd_t *tsd; 1587 szind_t ind; 1588 1589 if (slow_path && unlikely(malloc_init())) { 1590 *tsdn = NULL; 1591 return (NULL); 1592 } 1593 1594 tsd = tsd_fetch(); 1595 *tsdn = tsd_tsdn(tsd); 1596 witness_assert_lockless(tsd_tsdn(tsd)); 1597 1598 ind = size2index(size); 1599 if (unlikely(ind >= NSIZES)) 1600 return (NULL); 1601 1602 if (config_stats || (config_prof && opt_prof) || (slow_path && 1603 config_valgrind && unlikely(in_valgrind))) { 1604 *usize = index2size(ind); 1605 assert(*usize > 0 && *usize <= HUGE_MAXCLASS); 1606 } 1607 1608 if (config_prof && opt_prof) 1609 return (ialloc_prof(tsd, *usize, ind, zero, slow_path)); 1610 1611 return (ialloc(tsd, size, ind, zero, slow_path)); 1612 } 1613 1614 JEMALLOC_ALWAYS_INLINE_C void 1615 ialloc_post_check(void *ret, tsdn_t *tsdn, size_t usize, const char *func, 1616 bool update_errno, bool slow_path) 1617 { 1618 1619 assert(!tsdn_null(tsdn) || ret == NULL); 1620 1621 if (unlikely(ret == NULL)) { 1622 if (slow_path && config_xmalloc && unlikely(opt_xmalloc)) { 1623 malloc_printf("<jemalloc>: Error in %s(): out of " 1624 "memory\n", func); 1625 abort(); 1626 } 1627 if (update_errno) 1628 set_errno(ENOMEM); 1629 } 1630 if (config_stats && likely(ret != NULL)) { 1631 assert(usize == isalloc(tsdn, ret, config_prof)); 1632 *tsd_thread_allocatedp_get(tsdn_tsd(tsdn)) += usize; 1633 } 1634 witness_assert_lockless(tsdn); 1635 } 1636 1637 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1638 void JEMALLOC_NOTHROW * 1639 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) 1640 je_malloc(size_t size) 1641 { 1642 void *ret; 1643 tsdn_t *tsdn; 1644 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1645 1646 if (size == 0) 1647 size = 1; 1648 1649 if (likely(!malloc_slow)) { 1650 ret = ialloc_body(size, false, &tsdn, &usize, false); 1651 ialloc_post_check(ret, tsdn, usize, "malloc", true, false); 1652 } else { 1653 ret = ialloc_body(size, false, &tsdn, &usize, true); 1654 ialloc_post_check(ret, tsdn, usize, "malloc", true, true); 1655 UTRACE(0, size, ret); 1656 JEMALLOC_VALGRIND_MALLOC(ret != NULL, tsdn, ret, usize, false); 1657 } 1658 1659 return (ret); 1660 } 1661 1662 static void * 1663 imemalign_prof_sample(tsd_t *tsd, size_t alignment, size_t usize, 1664 prof_tctx_t *tctx) 1665 { 1666 void *p; 1667 1668 if (tctx == NULL) 1669 return (NULL); 1670 if (usize <= SMALL_MAXCLASS) { 1671 assert(sa2u(LARGE_MINCLASS, alignment) == LARGE_MINCLASS); 1672 p = ipalloc(tsd, LARGE_MINCLASS, alignment, false); 1673 if (p == NULL) 1674 return (NULL); 1675 arena_prof_promoted(tsd_tsdn(tsd), p, usize); 1676 } else 1677 p = ipalloc(tsd, usize, alignment, false); 1678 1679 return (p); 1680 } 1681 1682 JEMALLOC_ALWAYS_INLINE_C void * 1683 imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize) 1684 { 1685 void *p; 1686 prof_tctx_t *tctx; 1687 1688 tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true); 1689 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1690 p = imemalign_prof_sample(tsd, alignment, usize, tctx); 1691 else 1692 p = ipalloc(tsd, usize, alignment, false); 1693 if (unlikely(p == NULL)) { 1694 prof_alloc_rollback(tsd, tctx, true); 1695 return (NULL); 1696 } 1697 prof_malloc(tsd_tsdn(tsd), p, usize, tctx); 1698 1699 return (p); 1700 } 1701 1702 JEMALLOC_ATTR(nonnull(1)) 1703 static int 1704 imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) 1705 { 1706 int ret; 1707 tsd_t *tsd; 1708 size_t usize; 1709 void *result; 1710 1711 assert(min_alignment != 0); 1712 1713 if (unlikely(malloc_init())) { 1714 tsd = NULL; 1715 result = NULL; 1716 goto label_oom; 1717 } 1718 tsd = tsd_fetch(); 1719 witness_assert_lockless(tsd_tsdn(tsd)); 1720 if (size == 0) 1721 size = 1; 1722 1723 /* Make sure that alignment is a large enough power of 2. */ 1724 if (unlikely(((alignment - 1) & alignment) != 0 1725 || (alignment < min_alignment))) { 1726 if (config_xmalloc && unlikely(opt_xmalloc)) { 1727 malloc_write("<jemalloc>: Error allocating " 1728 "aligned memory: invalid alignment\n"); 1729 abort(); 1730 } 1731 result = NULL; 1732 ret = EINVAL; 1733 goto label_return; 1734 } 1735 1736 usize = sa2u(size, alignment); 1737 if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) { 1738 result = NULL; 1739 goto label_oom; 1740 } 1741 1742 if (config_prof && opt_prof) 1743 result = imemalign_prof(tsd, alignment, usize); 1744 else 1745 result = ipalloc(tsd, usize, alignment, false); 1746 if (unlikely(result == NULL)) 1747 goto label_oom; 1748 assert(((uintptr_t)result & (alignment - 1)) == ZU(0)); 1749 1750 *memptr = result; 1751 ret = 0; 1752 label_return: 1753 if (config_stats && likely(result != NULL)) { 1754 assert(usize == isalloc(tsd_tsdn(tsd), result, config_prof)); 1755 *tsd_thread_allocatedp_get(tsd) += usize; 1756 } 1757 UTRACE(0, size, result); 1758 JEMALLOC_VALGRIND_MALLOC(result != NULL, tsd_tsdn(tsd), result, usize, 1759 false); 1760 witness_assert_lockless(tsd_tsdn(tsd)); 1761 return (ret); 1762 label_oom: 1763 assert(result == NULL); 1764 if (config_xmalloc && unlikely(opt_xmalloc)) { 1765 malloc_write("<jemalloc>: Error allocating aligned memory: " 1766 "out of memory\n"); 1767 abort(); 1768 } 1769 ret = ENOMEM; 1770 witness_assert_lockless(tsd_tsdn(tsd)); 1771 goto label_return; 1772 } 1773 1774 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 1775 JEMALLOC_ATTR(nonnull(1)) 1776 je_posix_memalign(void **memptr, size_t alignment, size_t size) 1777 { 1778 int ret; 1779 1780 ret = imemalign(memptr, alignment, size, sizeof(void *)); 1781 1782 return (ret); 1783 } 1784 1785 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1786 void JEMALLOC_NOTHROW * 1787 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(2) 1788 je_aligned_alloc(size_t alignment, size_t size) 1789 { 1790 void *ret; 1791 int err; 1792 1793 if (unlikely((err = imemalign(&ret, alignment, size, 1)) != 0)) { 1794 ret = NULL; 1795 set_errno(err); 1796 } 1797 1798 return (ret); 1799 } 1800 1801 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1802 void JEMALLOC_NOTHROW * 1803 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2) 1804 je_calloc(size_t num, size_t size) 1805 { 1806 void *ret; 1807 tsdn_t *tsdn; 1808 size_t num_size; 1809 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1810 1811 num_size = num * size; 1812 if (unlikely(num_size == 0)) { 1813 if (num == 0 || size == 0) 1814 num_size = 1; 1815 else 1816 num_size = HUGE_MAXCLASS + 1; /* Trigger OOM. */ 1817 /* 1818 * Try to avoid division here. We know that it isn't possible to 1819 * overflow during multiplication if neither operand uses any of the 1820 * most significant half of the bits in a size_t. 1821 */ 1822 } else if (unlikely(((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 1823 2))) && (num_size / size != num))) 1824 num_size = HUGE_MAXCLASS + 1; /* size_t overflow. */ 1825 1826 if (likely(!malloc_slow)) { 1827 ret = ialloc_body(num_size, true, &tsdn, &usize, false); 1828 ialloc_post_check(ret, tsdn, usize, "calloc", true, false); 1829 } else { 1830 ret = ialloc_body(num_size, true, &tsdn, &usize, true); 1831 ialloc_post_check(ret, tsdn, usize, "calloc", true, true); 1832 UTRACE(0, num_size, ret); 1833 JEMALLOC_VALGRIND_MALLOC(ret != NULL, tsdn, ret, usize, true); 1834 } 1835 1836 return (ret); 1837 } 1838 1839 static void * 1840 irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, 1841 prof_tctx_t *tctx) 1842 { 1843 void *p; 1844 1845 if (tctx == NULL) 1846 return (NULL); 1847 if (usize <= SMALL_MAXCLASS) { 1848 p = iralloc(tsd, old_ptr, old_usize, LARGE_MINCLASS, 0, false); 1849 if (p == NULL) 1850 return (NULL); 1851 arena_prof_promoted(tsd_tsdn(tsd), p, usize); 1852 } else 1853 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); 1854 1855 return (p); 1856 } 1857 1858 JEMALLOC_ALWAYS_INLINE_C void * 1859 irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize) 1860 { 1861 void *p; 1862 bool prof_active; 1863 prof_tctx_t *old_tctx, *tctx; 1864 1865 prof_active = prof_active_get_unlocked(); 1866 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr); 1867 tctx = prof_alloc_prep(tsd, usize, prof_active, true); 1868 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1869 p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx); 1870 else 1871 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); 1872 if (unlikely(p == NULL)) { 1873 prof_alloc_rollback(tsd, tctx, true); 1874 return (NULL); 1875 } 1876 prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize, 1877 old_tctx); 1878 1879 return (p); 1880 } 1881 1882 JEMALLOC_INLINE_C void 1883 ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) 1884 { 1885 size_t usize; 1886 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1887 1888 witness_assert_lockless(tsd_tsdn(tsd)); 1889 1890 assert(ptr != NULL); 1891 assert(malloc_initialized() || IS_INITIALIZER); 1892 1893 if (config_prof && opt_prof) { 1894 usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); 1895 prof_free(tsd, ptr, usize); 1896 } else if (config_stats || config_valgrind) 1897 usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); 1898 if (config_stats) 1899 *tsd_thread_deallocatedp_get(tsd) += usize; 1900 1901 if (likely(!slow_path)) 1902 iqalloc(tsd, ptr, tcache, false); 1903 else { 1904 if (config_valgrind && unlikely(in_valgrind)) 1905 rzsize = p2rz(tsd_tsdn(tsd), ptr); 1906 iqalloc(tsd, ptr, tcache, true); 1907 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1908 } 1909 } 1910 1911 JEMALLOC_INLINE_C void 1912 isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) 1913 { 1914 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1915 1916 witness_assert_lockless(tsd_tsdn(tsd)); 1917 1918 assert(ptr != NULL); 1919 assert(malloc_initialized() || IS_INITIALIZER); 1920 1921 if (config_prof && opt_prof) 1922 prof_free(tsd, ptr, usize); 1923 if (config_stats) 1924 *tsd_thread_deallocatedp_get(tsd) += usize; 1925 if (config_valgrind && unlikely(in_valgrind)) 1926 rzsize = p2rz(tsd_tsdn(tsd), ptr); 1927 isqalloc(tsd, ptr, usize, tcache, slow_path); 1928 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1929 } 1930 1931 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1932 void JEMALLOC_NOTHROW * 1933 JEMALLOC_ALLOC_SIZE(2) 1934 je_realloc(void *ptr, size_t size) 1935 { 1936 void *ret; 1937 tsdn_t *tsdn JEMALLOC_CC_SILENCE_INIT(NULL); 1938 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1939 size_t old_usize = 0; 1940 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1941 1942 if (unlikely(size == 0)) { 1943 if (ptr != NULL) { 1944 tsd_t *tsd; 1945 1946 /* realloc(ptr, 0) is equivalent to free(ptr). */ 1947 UTRACE(ptr, 0, 0); 1948 tsd = tsd_fetch(); 1949 ifree(tsd, ptr, tcache_get(tsd, false), true); 1950 return (NULL); 1951 } 1952 size = 1; 1953 } 1954 1955 if (likely(ptr != NULL)) { 1956 tsd_t *tsd; 1957 1958 assert(malloc_initialized() || IS_INITIALIZER); 1959 malloc_thread_init(); 1960 tsd = tsd_fetch(); 1961 1962 witness_assert_lockless(tsd_tsdn(tsd)); 1963 1964 old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); 1965 if (config_valgrind && unlikely(in_valgrind)) { 1966 old_rzsize = config_prof ? p2rz(tsd_tsdn(tsd), ptr) : 1967 u2rz(old_usize); 1968 } 1969 1970 if (config_prof && opt_prof) { 1971 usize = s2u(size); 1972 ret = unlikely(usize == 0 || usize > HUGE_MAXCLASS) ? 1973 NULL : irealloc_prof(tsd, ptr, old_usize, usize); 1974 } else { 1975 if (config_stats || (config_valgrind && 1976 unlikely(in_valgrind))) 1977 usize = s2u(size); 1978 ret = iralloc(tsd, ptr, old_usize, size, 0, false); 1979 } 1980 tsdn = tsd_tsdn(tsd); 1981 } else { 1982 /* realloc(NULL, size) is equivalent to malloc(size). */ 1983 if (likely(!malloc_slow)) 1984 ret = ialloc_body(size, false, &tsdn, &usize, false); 1985 else 1986 ret = ialloc_body(size, false, &tsdn, &usize, true); 1987 assert(!tsdn_null(tsdn) || ret == NULL); 1988 } 1989 1990 if (unlikely(ret == NULL)) { 1991 if (config_xmalloc && unlikely(opt_xmalloc)) { 1992 malloc_write("<jemalloc>: Error in realloc(): " 1993 "out of memory\n"); 1994 abort(); 1995 } 1996 set_errno(ENOMEM); 1997 } 1998 if (config_stats && likely(ret != NULL)) { 1999 tsd_t *tsd; 2000 2001 assert(usize == isalloc(tsdn, ret, config_prof)); 2002 tsd = tsdn_tsd(tsdn); 2003 *tsd_thread_allocatedp_get(tsd) += usize; 2004 *tsd_thread_deallocatedp_get(tsd) += old_usize; 2005 } 2006 UTRACE(ptr, size, ret); 2007 JEMALLOC_VALGRIND_REALLOC(maybe, tsdn, ret, usize, maybe, ptr, 2008 old_usize, old_rzsize, maybe, false); 2009 witness_assert_lockless(tsdn); 2010 return (ret); 2011 } 2012 2013 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 2014 je_free(void *ptr) 2015 { 2016 2017 UTRACE(ptr, 0, 0); 2018 if (likely(ptr != NULL)) { 2019 tsd_t *tsd = tsd_fetch(); 2020 witness_assert_lockless(tsd_tsdn(tsd)); 2021 if (likely(!malloc_slow)) 2022 ifree(tsd, ptr, tcache_get(tsd, false), false); 2023 else 2024 ifree(tsd, ptr, tcache_get(tsd, false), true); 2025 witness_assert_lockless(tsd_tsdn(tsd)); 2026 } 2027 } 2028 2029 /* 2030 * End malloc(3)-compatible functions. 2031 */ 2032 /******************************************************************************/ 2033 /* 2034 * Begin non-standard override functions. 2035 */ 2036 2037 #ifdef JEMALLOC_OVERRIDE_MEMALIGN 2038 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2039 void JEMALLOC_NOTHROW * 2040 JEMALLOC_ATTR(malloc) 2041 je_memalign(size_t alignment, size_t size) 2042 { 2043 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 2044 if (unlikely(imemalign(&ret, alignment, size, 1) != 0)) 2045 ret = NULL; 2046 return (ret); 2047 } 2048 #endif 2049 2050 #ifdef JEMALLOC_OVERRIDE_VALLOC 2051 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2052 void JEMALLOC_NOTHROW * 2053 JEMALLOC_ATTR(malloc) 2054 je_valloc(size_t size) 2055 { 2056 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 2057 if (unlikely(imemalign(&ret, PAGE, size, 1) != 0)) 2058 ret = NULL; 2059 return (ret); 2060 } 2061 #endif 2062 2063 /* 2064 * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has 2065 * #define je_malloc malloc 2066 */ 2067 #define malloc_is_malloc 1 2068 #define is_malloc_(a) malloc_is_ ## a 2069 #define is_malloc(a) is_malloc_(a) 2070 2071 #if ((is_malloc(je_malloc) == 1) && defined(JEMALLOC_GLIBC_MALLOC_HOOK)) 2072 /* 2073 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible 2074 * to inconsistently reference libc's malloc(3)-compatible functions 2075 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541). 2076 * 2077 * These definitions interpose hooks in glibc. The functions are actually 2078 * passed an extra argument for the caller return address, which will be 2079 * ignored. 2080 */ 2081 JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free; 2082 JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc; 2083 JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc; 2084 # ifdef JEMALLOC_GLIBC_MEMALIGN_HOOK 2085 JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) = 2086 je_memalign; 2087 # endif 2088 2089 #ifdef CPU_COUNT 2090 /* 2091 * To enable static linking with glibc, the libc specific malloc interface must 2092 * be implemented also, so none of glibc's malloc.o functions are added to the 2093 * link. 2094 */ 2095 #define ALIAS(je_fn) __attribute__((alias (#je_fn), used)) 2096 /* To force macro expansion of je_ prefix before stringification. */ 2097 #define PREALIAS(je_fn) ALIAS(je_fn) 2098 void *__libc_malloc(size_t size) PREALIAS(je_malloc); 2099 void __libc_free(void* ptr) PREALIAS(je_free); 2100 void *__libc_realloc(void* ptr, size_t size) PREALIAS(je_realloc); 2101 void *__libc_calloc(size_t n, size_t size) PREALIAS(je_calloc); 2102 void *__libc_memalign(size_t align, size_t s) PREALIAS(je_memalign); 2103 void *__libc_valloc(size_t size) PREALIAS(je_valloc); 2104 int __posix_memalign(void** r, size_t a, size_t s) 2105 PREALIAS(je_posix_memalign); 2106 #undef PREALIAS 2107 #undef ALIAS 2108 2109 #endif 2110 2111 #endif 2112 2113 /* 2114 * End non-standard override functions. 2115 */ 2116 /******************************************************************************/ 2117 /* 2118 * Begin non-standard functions. 2119 */ 2120 2121 JEMALLOC_ALWAYS_INLINE_C bool 2122 imallocx_flags_decode(tsd_t *tsd, size_t size, int flags, size_t *usize, 2123 size_t *alignment, bool *zero, tcache_t **tcache, arena_t **arena) 2124 { 2125 2126 if ((flags & MALLOCX_LG_ALIGN_MASK) == 0) { 2127 *alignment = 0; 2128 *usize = s2u(size); 2129 } else { 2130 *alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags); 2131 *usize = sa2u(size, *alignment); 2132 } 2133 if (unlikely(*usize == 0 || *usize > HUGE_MAXCLASS)) 2134 return (true); 2135 *zero = MALLOCX_ZERO_GET(flags); 2136 if ((flags & MALLOCX_TCACHE_MASK) != 0) { 2137 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 2138 *tcache = NULL; 2139 else 2140 *tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2141 } else 2142 *tcache = tcache_get(tsd, true); 2143 if ((flags & MALLOCX_ARENA_MASK) != 0) { 2144 unsigned arena_ind = MALLOCX_ARENA_GET(flags); 2145 *arena = arena_get(tsd_tsdn(tsd), arena_ind, true); 2146 if (unlikely(*arena == NULL)) 2147 return (true); 2148 } else 2149 *arena = NULL; 2150 return (false); 2151 } 2152 2153 JEMALLOC_ALWAYS_INLINE_C void * 2154 imallocx_flags(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, 2155 tcache_t *tcache, arena_t *arena, bool slow_path) 2156 { 2157 szind_t ind; 2158 2159 if (unlikely(alignment != 0)) 2160 return (ipalloct(tsdn, usize, alignment, zero, tcache, arena)); 2161 ind = size2index(usize); 2162 assert(ind < NSIZES); 2163 return (iallocztm(tsdn, usize, ind, zero, tcache, false, arena, 2164 slow_path)); 2165 } 2166 2167 static void * 2168 imallocx_prof_sample(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, 2169 tcache_t *tcache, arena_t *arena, bool slow_path) 2170 { 2171 void *p; 2172 2173 if (usize <= SMALL_MAXCLASS) { 2174 assert(((alignment == 0) ? s2u(LARGE_MINCLASS) : 2175 sa2u(LARGE_MINCLASS, alignment)) == LARGE_MINCLASS); 2176 p = imallocx_flags(tsdn, LARGE_MINCLASS, alignment, zero, 2177 tcache, arena, slow_path); 2178 if (p == NULL) 2179 return (NULL); 2180 arena_prof_promoted(tsdn, p, usize); 2181 } else { 2182 p = imallocx_flags(tsdn, usize, alignment, zero, tcache, arena, 2183 slow_path); 2184 } 2185 2186 return (p); 2187 } 2188 2189 JEMALLOC_ALWAYS_INLINE_C void * 2190 imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize, bool slow_path) 2191 { 2192 void *p; 2193 size_t alignment; 2194 bool zero; 2195 tcache_t *tcache; 2196 arena_t *arena; 2197 prof_tctx_t *tctx; 2198 2199 if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment, 2200 &zero, &tcache, &arena))) 2201 return (NULL); 2202 tctx = prof_alloc_prep(tsd, *usize, prof_active_get_unlocked(), true); 2203 if (likely((uintptr_t)tctx == (uintptr_t)1U)) { 2204 p = imallocx_flags(tsd_tsdn(tsd), *usize, alignment, zero, 2205 tcache, arena, slow_path); 2206 } else if ((uintptr_t)tctx > (uintptr_t)1U) { 2207 p = imallocx_prof_sample(tsd_tsdn(tsd), *usize, alignment, zero, 2208 tcache, arena, slow_path); 2209 } else 2210 p = NULL; 2211 if (unlikely(p == NULL)) { 2212 prof_alloc_rollback(tsd, tctx, true); 2213 return (NULL); 2214 } 2215 prof_malloc(tsd_tsdn(tsd), p, *usize, tctx); 2216 2217 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0)); 2218 return (p); 2219 } 2220 2221 JEMALLOC_ALWAYS_INLINE_C void * 2222 imallocx_no_prof(tsd_t *tsd, size_t size, int flags, size_t *usize, 2223 bool slow_path) 2224 { 2225 void *p; 2226 size_t alignment; 2227 bool zero; 2228 tcache_t *tcache; 2229 arena_t *arena; 2230 2231 if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment, 2232 &zero, &tcache, &arena))) 2233 return (NULL); 2234 p = imallocx_flags(tsd_tsdn(tsd), *usize, alignment, zero, tcache, 2235 arena, slow_path); 2236 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0)); 2237 return (p); 2238 } 2239 2240 /* This function guarantees that *tsdn is non-NULL on success. */ 2241 JEMALLOC_ALWAYS_INLINE_C void * 2242 imallocx_body(size_t size, int flags, tsdn_t **tsdn, size_t *usize, 2243 bool slow_path) 2244 { 2245 tsd_t *tsd; 2246 2247 if (slow_path && unlikely(malloc_init())) { 2248 *tsdn = NULL; 2249 return (NULL); 2250 } 2251 2252 tsd = tsd_fetch(); 2253 *tsdn = tsd_tsdn(tsd); 2254 witness_assert_lockless(tsd_tsdn(tsd)); 2255 2256 if (likely(flags == 0)) { 2257 szind_t ind = size2index(size); 2258 if (unlikely(ind >= NSIZES)) 2259 return (NULL); 2260 if (config_stats || (config_prof && opt_prof) || (slow_path && 2261 config_valgrind && unlikely(in_valgrind))) { 2262 *usize = index2size(ind); 2263 assert(*usize > 0 && *usize <= HUGE_MAXCLASS); 2264 } 2265 2266 if (config_prof && opt_prof) { 2267 return (ialloc_prof(tsd, *usize, ind, false, 2268 slow_path)); 2269 } 2270 2271 return (ialloc(tsd, size, ind, false, slow_path)); 2272 } 2273 2274 if (config_prof && opt_prof) 2275 return (imallocx_prof(tsd, size, flags, usize, slow_path)); 2276 2277 return (imallocx_no_prof(tsd, size, flags, usize, slow_path)); 2278 } 2279 2280 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2281 void JEMALLOC_NOTHROW * 2282 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) 2283 je_mallocx(size_t size, int flags) 2284 { 2285 tsdn_t *tsdn; 2286 void *p; 2287 size_t usize; 2288 2289 assert(size != 0); 2290 2291 if (likely(!malloc_slow)) { 2292 p = imallocx_body(size, flags, &tsdn, &usize, false); 2293 ialloc_post_check(p, tsdn, usize, "mallocx", false, false); 2294 } else { 2295 p = imallocx_body(size, flags, &tsdn, &usize, true); 2296 ialloc_post_check(p, tsdn, usize, "mallocx", false, true); 2297 UTRACE(0, size, p); 2298 JEMALLOC_VALGRIND_MALLOC(p != NULL, tsdn, p, usize, 2299 MALLOCX_ZERO_GET(flags)); 2300 } 2301 2302 return (p); 2303 } 2304 2305 static void * 2306 irallocx_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, 2307 size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena, 2308 prof_tctx_t *tctx) 2309 { 2310 void *p; 2311 2312 if (tctx == NULL) 2313 return (NULL); 2314 if (usize <= SMALL_MAXCLASS) { 2315 p = iralloct(tsd, old_ptr, old_usize, LARGE_MINCLASS, alignment, 2316 zero, tcache, arena); 2317 if (p == NULL) 2318 return (NULL); 2319 arena_prof_promoted(tsd_tsdn(tsd), p, usize); 2320 } else { 2321 p = iralloct(tsd, old_ptr, old_usize, usize, alignment, zero, 2322 tcache, arena); 2323 } 2324 2325 return (p); 2326 } 2327 2328 JEMALLOC_ALWAYS_INLINE_C void * 2329 irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, 2330 size_t alignment, size_t *usize, bool zero, tcache_t *tcache, 2331 arena_t *arena) 2332 { 2333 void *p; 2334 bool prof_active; 2335 prof_tctx_t *old_tctx, *tctx; 2336 2337 prof_active = prof_active_get_unlocked(); 2338 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr); 2339 tctx = prof_alloc_prep(tsd, *usize, prof_active, false); 2340 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { 2341 p = irallocx_prof_sample(tsd, old_ptr, old_usize, *usize, 2342 alignment, zero, tcache, arena, tctx); 2343 } else { 2344 p = iralloct(tsd, old_ptr, old_usize, size, alignment, zero, 2345 tcache, arena); 2346 } 2347 if (unlikely(p == NULL)) { 2348 prof_alloc_rollback(tsd, tctx, false); 2349 return (NULL); 2350 } 2351 2352 if (p == old_ptr && alignment != 0) { 2353 /* 2354 * The allocation did not move, so it is possible that the size 2355 * class is smaller than would guarantee the requested 2356 * alignment, and that the alignment constraint was 2357 * serendipitously satisfied. Additionally, old_usize may not 2358 * be the same as the current usize because of in-place large 2359 * reallocation. Therefore, query the actual value of usize. 2360 */ 2361 *usize = isalloc(tsd_tsdn(tsd), p, config_prof); 2362 } 2363 prof_realloc(tsd, p, *usize, tctx, prof_active, false, old_ptr, 2364 old_usize, old_tctx); 2365 2366 return (p); 2367 } 2368 2369 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2370 void JEMALLOC_NOTHROW * 2371 JEMALLOC_ALLOC_SIZE(2) 2372 je_rallocx(void *ptr, size_t size, int flags) 2373 { 2374 void *p; 2375 tsd_t *tsd; 2376 size_t usize; 2377 size_t old_usize; 2378 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 2379 size_t alignment = MALLOCX_ALIGN_GET(flags); 2380 bool zero = flags & MALLOCX_ZERO; 2381 arena_t *arena; 2382 tcache_t *tcache; 2383 2384 assert(ptr != NULL); 2385 assert(size != 0); 2386 assert(malloc_initialized() || IS_INITIALIZER); 2387 malloc_thread_init(); 2388 tsd = tsd_fetch(); 2389 witness_assert_lockless(tsd_tsdn(tsd)); 2390 2391 if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { 2392 unsigned arena_ind = MALLOCX_ARENA_GET(flags); 2393 arena = arena_get(tsd_tsdn(tsd), arena_ind, true); 2394 if (unlikely(arena == NULL)) 2395 goto label_oom; 2396 } else 2397 arena = NULL; 2398 2399 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 2400 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 2401 tcache = NULL; 2402 else 2403 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2404 } else 2405 tcache = tcache_get(tsd, true); 2406 2407 old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); 2408 if (config_valgrind && unlikely(in_valgrind)) 2409 old_rzsize = u2rz(old_usize); 2410 2411 if (config_prof && opt_prof) { 2412 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 2413 if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) 2414 goto label_oom; 2415 p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize, 2416 zero, tcache, arena); 2417 if (unlikely(p == NULL)) 2418 goto label_oom; 2419 } else { 2420 p = iralloct(tsd, ptr, old_usize, size, alignment, zero, 2421 tcache, arena); 2422 if (unlikely(p == NULL)) 2423 goto label_oom; 2424 if (config_stats || (config_valgrind && unlikely(in_valgrind))) 2425 usize = isalloc(tsd_tsdn(tsd), p, config_prof); 2426 } 2427 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0)); 2428 2429 if (config_stats) { 2430 *tsd_thread_allocatedp_get(tsd) += usize; 2431 *tsd_thread_deallocatedp_get(tsd) += old_usize; 2432 } 2433 UTRACE(ptr, size, p); 2434 JEMALLOC_VALGRIND_REALLOC(maybe, tsd_tsdn(tsd), p, usize, no, ptr, 2435 old_usize, old_rzsize, no, zero); 2436 witness_assert_lockless(tsd_tsdn(tsd)); 2437 return (p); 2438 label_oom: 2439 if (config_xmalloc && unlikely(opt_xmalloc)) { 2440 malloc_write("<jemalloc>: Error in rallocx(): out of memory\n"); 2441 abort(); 2442 } 2443 UTRACE(ptr, size, 0); 2444 witness_assert_lockless(tsd_tsdn(tsd)); 2445 return (NULL); 2446 } 2447 2448 JEMALLOC_ALWAYS_INLINE_C size_t 2449 ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size, 2450 size_t extra, size_t alignment, bool zero) 2451 { 2452 size_t usize; 2453 2454 if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero)) 2455 return (old_usize); 2456 usize = isalloc(tsdn, ptr, config_prof); 2457 2458 return (usize); 2459 } 2460 2461 static size_t 2462 ixallocx_prof_sample(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size, 2463 size_t extra, size_t alignment, bool zero, prof_tctx_t *tctx) 2464 { 2465 size_t usize; 2466 2467 if (tctx == NULL) 2468 return (old_usize); 2469 usize = ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment, 2470 zero); 2471 2472 return (usize); 2473 } 2474 2475 JEMALLOC_ALWAYS_INLINE_C size_t 2476 ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, 2477 size_t extra, size_t alignment, bool zero) 2478 { 2479 size_t usize_max, usize; 2480 bool prof_active; 2481 prof_tctx_t *old_tctx, *tctx; 2482 2483 prof_active = prof_active_get_unlocked(); 2484 old_tctx = prof_tctx_get(tsd_tsdn(tsd), ptr); 2485 /* 2486 * usize isn't knowable before ixalloc() returns when extra is non-zero. 2487 * Therefore, compute its maximum possible value and use that in 2488 * prof_alloc_prep() to decide whether to capture a backtrace. 2489 * prof_realloc() will use the actual usize to decide whether to sample. 2490 */ 2491 if (alignment == 0) { 2492 usize_max = s2u(size+extra); 2493 assert(usize_max > 0 && usize_max <= HUGE_MAXCLASS); 2494 } else { 2495 usize_max = sa2u(size+extra, alignment); 2496 if (unlikely(usize_max == 0 || usize_max > HUGE_MAXCLASS)) { 2497 /* 2498 * usize_max is out of range, and chances are that 2499 * allocation will fail, but use the maximum possible 2500 * value and carry on with prof_alloc_prep(), just in 2501 * case allocation succeeds. 2502 */ 2503 usize_max = HUGE_MAXCLASS; 2504 } 2505 } 2506 tctx = prof_alloc_prep(tsd, usize_max, prof_active, false); 2507 2508 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { 2509 usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize, 2510 size, extra, alignment, zero, tctx); 2511 } else { 2512 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size, 2513 extra, alignment, zero); 2514 } 2515 if (usize == old_usize) { 2516 prof_alloc_rollback(tsd, tctx, false); 2517 return (usize); 2518 } 2519 prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize, 2520 old_tctx); 2521 2522 return (usize); 2523 } 2524 2525 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 2526 je_xallocx(void *ptr, size_t size, size_t extra, int flags) 2527 { 2528 tsd_t *tsd; 2529 size_t usize, old_usize; 2530 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 2531 size_t alignment = MALLOCX_ALIGN_GET(flags); 2532 bool zero = flags & MALLOCX_ZERO; 2533 2534 assert(ptr != NULL); 2535 assert(size != 0); 2536 assert(SIZE_T_MAX - size >= extra); 2537 assert(malloc_initialized() || IS_INITIALIZER); 2538 malloc_thread_init(); 2539 tsd = tsd_fetch(); 2540 witness_assert_lockless(tsd_tsdn(tsd)); 2541 2542 old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); 2543 2544 /* 2545 * The API explicitly absolves itself of protecting against (size + 2546 * extra) numerical overflow, but we may need to clamp extra to avoid 2547 * exceeding HUGE_MAXCLASS. 2548 * 2549 * Ordinarily, size limit checking is handled deeper down, but here we 2550 * have to check as part of (size + extra) clamping, since we need the 2551 * clamped value in the above helper functions. 2552 */ 2553 if (unlikely(size > HUGE_MAXCLASS)) { 2554 usize = old_usize; 2555 goto label_not_resized; 2556 } 2557 if (unlikely(HUGE_MAXCLASS - size < extra)) 2558 extra = HUGE_MAXCLASS - size; 2559 2560 if (config_valgrind && unlikely(in_valgrind)) 2561 old_rzsize = u2rz(old_usize); 2562 2563 if (config_prof && opt_prof) { 2564 usize = ixallocx_prof(tsd, ptr, old_usize, size, extra, 2565 alignment, zero); 2566 } else { 2567 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size, 2568 extra, alignment, zero); 2569 } 2570 if (unlikely(usize == old_usize)) 2571 goto label_not_resized; 2572 2573 if (config_stats) { 2574 *tsd_thread_allocatedp_get(tsd) += usize; 2575 *tsd_thread_deallocatedp_get(tsd) += old_usize; 2576 } 2577 JEMALLOC_VALGRIND_REALLOC(no, tsd_tsdn(tsd), ptr, usize, no, ptr, 2578 old_usize, old_rzsize, no, zero); 2579 label_not_resized: 2580 UTRACE(ptr, size, ptr); 2581 witness_assert_lockless(tsd_tsdn(tsd)); 2582 return (usize); 2583 } 2584 2585 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 2586 JEMALLOC_ATTR(pure) 2587 je_sallocx(const void *ptr, int flags) 2588 { 2589 size_t usize; 2590 tsdn_t *tsdn; 2591 2592 assert(malloc_initialized() || IS_INITIALIZER); 2593 malloc_thread_init(); 2594 2595 tsdn = tsdn_fetch(); 2596 witness_assert_lockless(tsdn); 2597 2598 if (config_ivsalloc) 2599 usize = ivsalloc(tsdn, ptr, config_prof); 2600 else 2601 usize = isalloc(tsdn, ptr, config_prof); 2602 2603 witness_assert_lockless(tsdn); 2604 return (usize); 2605 } 2606 2607 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 2608 je_dallocx(void *ptr, int flags) 2609 { 2610 tsd_t *tsd; 2611 tcache_t *tcache; 2612 2613 assert(ptr != NULL); 2614 assert(malloc_initialized() || IS_INITIALIZER); 2615 2616 tsd = tsd_fetch(); 2617 witness_assert_lockless(tsd_tsdn(tsd)); 2618 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 2619 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 2620 tcache = NULL; 2621 else 2622 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2623 } else 2624 tcache = tcache_get(tsd, false); 2625 2626 UTRACE(ptr, 0, 0); 2627 if (likely(!malloc_slow)) 2628 ifree(tsd, ptr, tcache, false); 2629 else 2630 ifree(tsd, ptr, tcache, true); 2631 witness_assert_lockless(tsd_tsdn(tsd)); 2632 } 2633 2634 JEMALLOC_ALWAYS_INLINE_C size_t 2635 inallocx(tsdn_t *tsdn, size_t size, int flags) 2636 { 2637 size_t usize; 2638 2639 witness_assert_lockless(tsdn); 2640 2641 if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) 2642 usize = s2u(size); 2643 else 2644 usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags)); 2645 witness_assert_lockless(tsdn); 2646 return (usize); 2647 } 2648 2649 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 2650 je_sdallocx(void *ptr, size_t size, int flags) 2651 { 2652 tsd_t *tsd; 2653 tcache_t *tcache; 2654 size_t usize; 2655 2656 assert(ptr != NULL); 2657 assert(malloc_initialized() || IS_INITIALIZER); 2658 tsd = tsd_fetch(); 2659 usize = inallocx(tsd_tsdn(tsd), size, flags); 2660 assert(usize == isalloc(tsd_tsdn(tsd), ptr, config_prof)); 2661 2662 witness_assert_lockless(tsd_tsdn(tsd)); 2663 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 2664 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 2665 tcache = NULL; 2666 else 2667 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2668 } else 2669 tcache = tcache_get(tsd, false); 2670 2671 UTRACE(ptr, 0, 0); 2672 if (likely(!malloc_slow)) 2673 isfree(tsd, ptr, usize, tcache, false); 2674 else 2675 isfree(tsd, ptr, usize, tcache, true); 2676 witness_assert_lockless(tsd_tsdn(tsd)); 2677 } 2678 2679 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 2680 JEMALLOC_ATTR(pure) 2681 je_nallocx(size_t size, int flags) 2682 { 2683 size_t usize; 2684 tsdn_t *tsdn; 2685 2686 assert(size != 0); 2687 2688 if (unlikely(malloc_init())) 2689 return (0); 2690 2691 tsdn = tsdn_fetch(); 2692 witness_assert_lockless(tsdn); 2693 2694 usize = inallocx(tsdn, size, flags); 2695 if (unlikely(usize > HUGE_MAXCLASS)) 2696 return (0); 2697 2698 witness_assert_lockless(tsdn); 2699 return (usize); 2700 } 2701 2702 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 2703 je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, 2704 size_t newlen) 2705 { 2706 int ret; 2707 tsd_t *tsd; 2708 2709 if (unlikely(malloc_init())) 2710 return (EAGAIN); 2711 2712 tsd = tsd_fetch(); 2713 witness_assert_lockless(tsd_tsdn(tsd)); 2714 ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen); 2715 witness_assert_lockless(tsd_tsdn(tsd)); 2716 return (ret); 2717 } 2718 2719 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 2720 je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) 2721 { 2722 int ret; 2723 tsdn_t *tsdn; 2724 2725 if (unlikely(malloc_init())) 2726 return (EAGAIN); 2727 2728 tsdn = tsdn_fetch(); 2729 witness_assert_lockless(tsdn); 2730 ret = ctl_nametomib(tsdn, name, mibp, miblenp); 2731 witness_assert_lockless(tsdn); 2732 return (ret); 2733 } 2734 2735 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 2736 je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 2737 void *newp, size_t newlen) 2738 { 2739 int ret; 2740 tsd_t *tsd; 2741 2742 if (unlikely(malloc_init())) 2743 return (EAGAIN); 2744 2745 tsd = tsd_fetch(); 2746 witness_assert_lockless(tsd_tsdn(tsd)); 2747 ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen); 2748 witness_assert_lockless(tsd_tsdn(tsd)); 2749 return (ret); 2750 } 2751 2752 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 2753 je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, 2754 const char *opts) 2755 { 2756 tsdn_t *tsdn; 2757 2758 tsdn = tsdn_fetch(); 2759 witness_assert_lockless(tsdn); 2760 stats_print(write_cb, cbopaque, opts); 2761 witness_assert_lockless(tsdn); 2762 } 2763 2764 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 2765 je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) 2766 { 2767 size_t ret; 2768 tsdn_t *tsdn; 2769 2770 assert(malloc_initialized() || IS_INITIALIZER); 2771 malloc_thread_init(); 2772 2773 tsdn = tsdn_fetch(); 2774 witness_assert_lockless(tsdn); 2775 2776 if (config_ivsalloc) 2777 ret = ivsalloc(tsdn, ptr, config_prof); 2778 else 2779 ret = (ptr == NULL) ? 0 : isalloc(tsdn, ptr, config_prof); 2780 2781 witness_assert_lockless(tsdn); 2782 return (ret); 2783 } 2784 2785 /* 2786 * End non-standard functions. 2787 */ 2788 /******************************************************************************/ 2789 /* 2790 * The following functions are used by threading libraries for protection of 2791 * malloc during fork(). 2792 */ 2793 2794 /* 2795 * If an application creates a thread before doing any allocation in the main 2796 * thread, then calls fork(2) in the main thread followed by memory allocation 2797 * in the child process, a race can occur that results in deadlock within the 2798 * child: the main thread may have forked while the created thread had 2799 * partially initialized the allocator. Ordinarily jemalloc prevents 2800 * fork/malloc races via the following functions it registers during 2801 * initialization using pthread_atfork(), but of course that does no good if 2802 * the allocator isn't fully initialized at fork time. The following library 2803 * constructor is a partial solution to this problem. It may still be possible 2804 * to trigger the deadlock described above, but doing so would involve forking 2805 * via a library constructor that runs before jemalloc's runs. 2806 */ 2807 #ifndef JEMALLOC_JET 2808 JEMALLOC_ATTR(constructor) 2809 static void 2810 jemalloc_constructor(void) 2811 { 2812 2813 malloc_init(); 2814 } 2815 #endif 2816 2817 #ifndef JEMALLOC_MUTEX_INIT_CB 2818 void 2819 jemalloc_prefork(void) 2820 #else 2821 JEMALLOC_EXPORT void 2822 _malloc_prefork(void) 2823 #endif 2824 { 2825 tsd_t *tsd; 2826 unsigned i, j, narenas; 2827 arena_t *arena; 2828 2829 #ifdef JEMALLOC_MUTEX_INIT_CB 2830 if (!malloc_initialized()) 2831 return; 2832 #endif 2833 assert(malloc_initialized()); 2834 2835 tsd = tsd_fetch(); 2836 2837 narenas = narenas_total_get(); 2838 2839 witness_prefork(tsd); 2840 /* Acquire all mutexes in a safe order. */ 2841 ctl_prefork(tsd_tsdn(tsd)); 2842 tcache_prefork(tsd_tsdn(tsd)); 2843 malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock); 2844 prof_prefork0(tsd_tsdn(tsd)); 2845 for (i = 0; i < 3; i++) { 2846 for (j = 0; j < narenas; j++) { 2847 if ((arena = arena_get(tsd_tsdn(tsd), j, false)) != 2848 NULL) { 2849 switch (i) { 2850 case 0: 2851 arena_prefork0(tsd_tsdn(tsd), arena); 2852 break; 2853 case 1: 2854 arena_prefork1(tsd_tsdn(tsd), arena); 2855 break; 2856 case 2: 2857 arena_prefork2(tsd_tsdn(tsd), arena); 2858 break; 2859 default: not_reached(); 2860 } 2861 } 2862 } 2863 } 2864 base_prefork(tsd_tsdn(tsd)); 2865 for (i = 0; i < narenas; i++) { 2866 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) 2867 arena_prefork3(tsd_tsdn(tsd), arena); 2868 } 2869 prof_prefork1(tsd_tsdn(tsd)); 2870 } 2871 2872 #ifndef JEMALLOC_MUTEX_INIT_CB 2873 void 2874 jemalloc_postfork_parent(void) 2875 #else 2876 JEMALLOC_EXPORT void 2877 _malloc_postfork(void) 2878 #endif 2879 { 2880 tsd_t *tsd; 2881 unsigned i, narenas; 2882 2883 #ifdef JEMALLOC_MUTEX_INIT_CB 2884 if (!malloc_initialized()) 2885 return; 2886 #endif 2887 assert(malloc_initialized()); 2888 2889 tsd = tsd_fetch(); 2890 2891 witness_postfork_parent(tsd); 2892 /* Release all mutexes, now that fork() has completed. */ 2893 base_postfork_parent(tsd_tsdn(tsd)); 2894 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 2895 arena_t *arena; 2896 2897 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) 2898 arena_postfork_parent(tsd_tsdn(tsd), arena); 2899 } 2900 prof_postfork_parent(tsd_tsdn(tsd)); 2901 malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock); 2902 tcache_postfork_parent(tsd_tsdn(tsd)); 2903 ctl_postfork_parent(tsd_tsdn(tsd)); 2904 } 2905 2906 void 2907 jemalloc_postfork_child(void) 2908 { 2909 tsd_t *tsd; 2910 unsigned i, narenas; 2911 2912 assert(malloc_initialized()); 2913 2914 tsd = tsd_fetch(); 2915 2916 witness_postfork_child(tsd); 2917 /* Release all mutexes, now that fork() has completed. */ 2918 base_postfork_child(tsd_tsdn(tsd)); 2919 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 2920 arena_t *arena; 2921 2922 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) 2923 arena_postfork_child(tsd_tsdn(tsd), arena); 2924 } 2925 prof_postfork_child(tsd_tsdn(tsd)); 2926 malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock); 2927 tcache_postfork_child(tsd_tsdn(tsd)); 2928 ctl_postfork_child(tsd_tsdn(tsd)); 2929 } 2930 2931 /******************************************************************************/ 2932 2933 /* ANDROID extension */ 2934 #include "android_je_iterate.c" 2935 #include "android_je_mallinfo.c" 2936 /* End ANDROID extension */ 2937
