1 #define JEMALLOC_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 4 /******************************************************************************/ 5 /* Data. */ 6 7 /* Runtime configuration options. */ 8 const char *je_malloc_conf JEMALLOC_ATTR(weak); 9 bool opt_abort = 10 #ifdef JEMALLOC_DEBUG 11 true 12 #else 13 false 14 #endif 15 ; 16 const char *opt_junk = 17 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 18 "true" 19 #else 20 "false" 21 #endif 22 ; 23 bool opt_junk_alloc = 24 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 25 true 26 #else 27 false 28 #endif 29 ; 30 bool opt_junk_free = 31 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 32 true 33 #else 34 false 35 #endif 36 ; 37 38 size_t opt_quarantine = ZU(0); 39 bool opt_redzone = false; 40 bool opt_utrace = false; 41 bool opt_xmalloc = false; 42 bool opt_zero = false; 43 size_t opt_narenas = 0; 44 45 /* Initialized to true if the process is running inside Valgrind. */ 46 bool in_valgrind; 47 48 unsigned ncpus; 49 50 /* Protects arenas initialization (arenas, narenas_total). */ 51 static malloc_mutex_t arenas_lock; 52 /* 53 * Arenas that are used to service external requests. Not all elements of the 54 * arenas array are necessarily used; arenas are created lazily as needed. 55 * 56 * arenas[0..narenas_auto) are used for automatic multiplexing of threads and 57 * arenas. arenas[narenas_auto..narenas_total) are only used if the application 58 * takes some action to create them and allocate from them. 59 */ 60 static arena_t **arenas; 61 static unsigned narenas_total; 62 static arena_t *a0; /* arenas[0]; read-only after initialization. */ 63 static unsigned narenas_auto; /* Read-only after initialization. */ 64 65 typedef enum { 66 malloc_init_uninitialized = 3, 67 malloc_init_a0_initialized = 2, 68 malloc_init_recursible = 1, 69 malloc_init_initialized = 0 /* Common case --> jnz. */ 70 } malloc_init_t; 71 static malloc_init_t malloc_init_state = malloc_init_uninitialized; 72 73 JEMALLOC_ALIGNED(CACHELINE) 74 const size_t index2size_tab[NSIZES] = { 75 #define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ 76 ((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)), 77 SIZE_CLASSES 78 #undef SC 79 }; 80 81 JEMALLOC_ALIGNED(CACHELINE) 82 const uint8_t size2index_tab[] = { 83 #if LG_TINY_MIN == 0 84 #warning "Dangerous LG_TINY_MIN" 85 #define S2B_0(i) i, 86 #elif LG_TINY_MIN == 1 87 #warning "Dangerous LG_TINY_MIN" 88 #define S2B_1(i) i, 89 #elif LG_TINY_MIN == 2 90 #warning "Dangerous LG_TINY_MIN" 91 #define S2B_2(i) i, 92 #elif LG_TINY_MIN == 3 93 #define S2B_3(i) i, 94 #elif LG_TINY_MIN == 4 95 #define S2B_4(i) i, 96 #elif LG_TINY_MIN == 5 97 #define S2B_5(i) i, 98 #elif LG_TINY_MIN == 6 99 #define S2B_6(i) i, 100 #elif LG_TINY_MIN == 7 101 #define S2B_7(i) i, 102 #elif LG_TINY_MIN == 8 103 #define S2B_8(i) i, 104 #elif LG_TINY_MIN == 9 105 #define S2B_9(i) i, 106 #elif LG_TINY_MIN == 10 107 #define S2B_10(i) i, 108 #elif LG_TINY_MIN == 11 109 #define S2B_11(i) i, 110 #else 111 #error "Unsupported LG_TINY_MIN" 112 #endif 113 #if LG_TINY_MIN < 1 114 #define S2B_1(i) S2B_0(i) S2B_0(i) 115 #endif 116 #if LG_TINY_MIN < 2 117 #define S2B_2(i) S2B_1(i) S2B_1(i) 118 #endif 119 #if LG_TINY_MIN < 3 120 #define S2B_3(i) S2B_2(i) S2B_2(i) 121 #endif 122 #if LG_TINY_MIN < 4 123 #define S2B_4(i) S2B_3(i) S2B_3(i) 124 #endif 125 #if LG_TINY_MIN < 5 126 #define S2B_5(i) S2B_4(i) S2B_4(i) 127 #endif 128 #if LG_TINY_MIN < 6 129 #define S2B_6(i) S2B_5(i) S2B_5(i) 130 #endif 131 #if LG_TINY_MIN < 7 132 #define S2B_7(i) S2B_6(i) S2B_6(i) 133 #endif 134 #if LG_TINY_MIN < 8 135 #define S2B_8(i) S2B_7(i) S2B_7(i) 136 #endif 137 #if LG_TINY_MIN < 9 138 #define S2B_9(i) S2B_8(i) S2B_8(i) 139 #endif 140 #if LG_TINY_MIN < 10 141 #define S2B_10(i) S2B_9(i) S2B_9(i) 142 #endif 143 #if LG_TINY_MIN < 11 144 #define S2B_11(i) S2B_10(i) S2B_10(i) 145 #endif 146 #define S2B_no(i) 147 #define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ 148 S2B_##lg_delta_lookup(index) 149 SIZE_CLASSES 150 #undef S2B_3 151 #undef S2B_4 152 #undef S2B_5 153 #undef S2B_6 154 #undef S2B_7 155 #undef S2B_8 156 #undef S2B_9 157 #undef S2B_10 158 #undef S2B_11 159 #undef S2B_no 160 #undef SC 161 }; 162 163 #ifdef JEMALLOC_THREADED_INIT 164 /* Used to let the initializing thread recursively allocate. */ 165 # define NO_INITIALIZER ((unsigned long)0) 166 # define INITIALIZER pthread_self() 167 # define IS_INITIALIZER (malloc_initializer == pthread_self()) 168 static pthread_t malloc_initializer = NO_INITIALIZER; 169 #else 170 # define NO_INITIALIZER false 171 # define INITIALIZER true 172 # define IS_INITIALIZER malloc_initializer 173 static bool malloc_initializer = NO_INITIALIZER; 174 #endif 175 176 /* Used to avoid initialization races. */ 177 #ifdef _WIN32 178 static malloc_mutex_t init_lock; 179 180 JEMALLOC_ATTR(constructor) 181 static void WINAPI 182 _init_init_lock(void) 183 { 184 185 malloc_mutex_init(&init_lock); 186 } 187 188 #ifdef _MSC_VER 189 # pragma section(".CRT$XCU", read) 190 JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used) 191 static const void (WINAPI *init_init_lock)(void) = _init_init_lock; 192 #endif 193 194 #else 195 static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; 196 #endif 197 198 typedef struct { 199 void *p; /* Input pointer (as in realloc(p, s)). */ 200 size_t s; /* Request size. */ 201 void *r; /* Result pointer. */ 202 } malloc_utrace_t; 203 204 #ifdef JEMALLOC_UTRACE 205 # define UTRACE(a, b, c) do { \ 206 if (unlikely(opt_utrace)) { \ 207 int utrace_serrno = errno; \ 208 malloc_utrace_t ut; \ 209 ut.p = (a); \ 210 ut.s = (b); \ 211 ut.r = (c); \ 212 utrace(&ut, sizeof(ut)); \ 213 errno = utrace_serrno; \ 214 } \ 215 } while (0) 216 #else 217 # define UTRACE(a, b, c) 218 #endif 219 220 /******************************************************************************/ 221 /* 222 * Function prototypes for static functions that are referenced prior to 223 * definition. 224 */ 225 226 static bool malloc_init_hard_a0(void); 227 static bool malloc_init_hard(void); 228 229 /******************************************************************************/ 230 /* 231 * Begin miscellaneous support functions. 232 */ 233 234 JEMALLOC_ALWAYS_INLINE_C bool 235 malloc_initialized(void) 236 { 237 238 return (malloc_init_state == malloc_init_initialized); 239 } 240 241 JEMALLOC_ALWAYS_INLINE_C void 242 malloc_thread_init(void) 243 { 244 245 /* 246 * TSD initialization can't be safely done as a side effect of 247 * deallocation, because it is possible for a thread to do nothing but 248 * deallocate its TLS data via free(), in which case writing to TLS 249 * would cause write-after-free memory corruption. The quarantine 250 * facility *only* gets used as a side effect of deallocation, so make 251 * a best effort attempt at initializing its TSD by hooking all 252 * allocation events. 253 */ 254 if (config_fill && unlikely(opt_quarantine)) 255 quarantine_alloc_hook(); 256 } 257 258 JEMALLOC_ALWAYS_INLINE_C bool 259 malloc_init_a0(void) 260 { 261 262 if (unlikely(malloc_init_state == malloc_init_uninitialized)) 263 return (malloc_init_hard_a0()); 264 return (false); 265 } 266 267 JEMALLOC_ALWAYS_INLINE_C bool 268 malloc_init(void) 269 { 270 271 if (unlikely(!malloc_initialized()) && malloc_init_hard()) 272 return (true); 273 malloc_thread_init(); 274 275 return (false); 276 } 277 278 /* 279 * The a0*() functions are used instead of i[mcd]alloc() in situations that 280 * cannot tolerate TLS variable access. 281 */ 282 283 arena_t * 284 a0get(void) 285 { 286 287 assert(a0 != NULL); 288 return (a0); 289 } 290 291 static void * 292 a0ialloc(size_t size, bool zero, bool is_metadata) 293 { 294 295 if (unlikely(malloc_init_a0())) 296 return (NULL); 297 298 return (iallocztm(NULL, size, zero, false, is_metadata, a0get())); 299 } 300 301 static void 302 a0idalloc(void *ptr, bool is_metadata) 303 { 304 305 idalloctm(NULL, ptr, false, is_metadata); 306 } 307 308 void * 309 a0malloc(size_t size) 310 { 311 312 return (a0ialloc(size, false, true)); 313 } 314 315 void 316 a0dalloc(void *ptr) 317 { 318 319 a0idalloc(ptr, true); 320 } 321 322 /* 323 * FreeBSD's libc uses the bootstrap_*() functions in bootstrap-senstive 324 * situations that cannot tolerate TLS variable access (TLS allocation and very 325 * early internal data structure initialization). 326 */ 327 328 void * 329 bootstrap_malloc(size_t size) 330 { 331 332 if (unlikely(size == 0)) 333 size = 1; 334 335 return (a0ialloc(size, false, false)); 336 } 337 338 void * 339 bootstrap_calloc(size_t num, size_t size) 340 { 341 size_t num_size; 342 343 num_size = num * size; 344 if (unlikely(num_size == 0)) { 345 assert(num == 0 || size == 0); 346 num_size = 1; 347 } 348 349 return (a0ialloc(num_size, true, false)); 350 } 351 352 void 353 bootstrap_free(void *ptr) 354 { 355 356 if (unlikely(ptr == NULL)) 357 return; 358 359 a0idalloc(ptr, false); 360 } 361 362 /* Create a new arena and insert it into the arenas array at index ind. */ 363 static arena_t * 364 arena_init_locked(unsigned ind) 365 { 366 arena_t *arena; 367 368 /* Expand arenas if necessary. */ 369 assert(ind <= narenas_total); 370 if (ind > MALLOCX_ARENA_MAX) 371 return (NULL); 372 if (ind == narenas_total) { 373 unsigned narenas_new = narenas_total + 1; 374 arena_t **arenas_new = 375 (arena_t **)a0malloc(CACHELINE_CEILING(narenas_new * 376 sizeof(arena_t *))); 377 if (arenas_new == NULL) 378 return (NULL); 379 memcpy(arenas_new, arenas, narenas_total * sizeof(arena_t *)); 380 arenas_new[ind] = NULL; 381 /* 382 * Deallocate only if arenas came from a0malloc() (not 383 * base_alloc()). 384 */ 385 if (narenas_total != narenas_auto) 386 a0dalloc(arenas); 387 arenas = arenas_new; 388 narenas_total = narenas_new; 389 } 390 391 /* 392 * Another thread may have already initialized arenas[ind] if it's an 393 * auto arena. 394 */ 395 arena = arenas[ind]; 396 if (arena != NULL) { 397 assert(ind < narenas_auto); 398 return (arena); 399 } 400 401 /* Actually initialize the arena. */ 402 arena = arenas[ind] = arena_new(ind); 403 return (arena); 404 } 405 406 arena_t * 407 arena_init(unsigned ind) 408 { 409 arena_t *arena; 410 411 malloc_mutex_lock(&arenas_lock); 412 arena = arena_init_locked(ind); 413 malloc_mutex_unlock(&arenas_lock); 414 return (arena); 415 } 416 417 unsigned 418 narenas_total_get(void) 419 { 420 unsigned narenas; 421 422 malloc_mutex_lock(&arenas_lock); 423 narenas = narenas_total; 424 malloc_mutex_unlock(&arenas_lock); 425 426 return (narenas); 427 } 428 429 static void 430 arena_bind_locked(tsd_t *tsd, unsigned ind) 431 { 432 arena_t *arena; 433 434 arena = arenas[ind]; 435 arena->nthreads++; 436 437 if (tsd_nominal(tsd)) 438 tsd_arena_set(tsd, arena); 439 } 440 441 static void 442 arena_bind(tsd_t *tsd, unsigned ind) 443 { 444 445 malloc_mutex_lock(&arenas_lock); 446 arena_bind_locked(tsd, ind); 447 malloc_mutex_unlock(&arenas_lock); 448 } 449 450 void 451 arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind) 452 { 453 arena_t *oldarena, *newarena; 454 455 malloc_mutex_lock(&arenas_lock); 456 oldarena = arenas[oldind]; 457 newarena = arenas[newind]; 458 oldarena->nthreads--; 459 newarena->nthreads++; 460 malloc_mutex_unlock(&arenas_lock); 461 tsd_arena_set(tsd, newarena); 462 } 463 464 unsigned 465 arena_nbound(unsigned ind) 466 { 467 unsigned nthreads; 468 469 malloc_mutex_lock(&arenas_lock); 470 nthreads = arenas[ind]->nthreads; 471 malloc_mutex_unlock(&arenas_lock); 472 return (nthreads); 473 } 474 475 static void 476 arena_unbind(tsd_t *tsd, unsigned ind) 477 { 478 arena_t *arena; 479 480 malloc_mutex_lock(&arenas_lock); 481 arena = arenas[ind]; 482 arena->nthreads--; 483 malloc_mutex_unlock(&arenas_lock); 484 tsd_arena_set(tsd, NULL); 485 } 486 487 arena_t * 488 arena_get_hard(tsd_t *tsd, unsigned ind, bool init_if_missing) 489 { 490 arena_t *arena; 491 arena_t **arenas_cache = tsd_arenas_cache_get(tsd); 492 unsigned narenas_cache = tsd_narenas_cache_get(tsd); 493 unsigned narenas_actual = narenas_total_get(); 494 495 /* Deallocate old cache if it's too small. */ 496 if (arenas_cache != NULL && narenas_cache < narenas_actual) { 497 a0dalloc(arenas_cache); 498 arenas_cache = NULL; 499 narenas_cache = 0; 500 tsd_arenas_cache_set(tsd, arenas_cache); 501 tsd_narenas_cache_set(tsd, narenas_cache); 502 } 503 504 /* Allocate cache if it's missing. */ 505 if (arenas_cache == NULL) { 506 bool *arenas_cache_bypassp = tsd_arenas_cache_bypassp_get(tsd); 507 assert(ind < narenas_actual || !init_if_missing); 508 narenas_cache = (ind < narenas_actual) ? narenas_actual : ind+1; 509 510 if (!*arenas_cache_bypassp) { 511 *arenas_cache_bypassp = true; 512 arenas_cache = (arena_t **)a0malloc(sizeof(arena_t *) * 513 narenas_cache); 514 *arenas_cache_bypassp = false; 515 } else 516 arenas_cache = NULL; 517 if (arenas_cache == NULL) { 518 /* 519 * This function must always tell the truth, even if 520 * it's slow, so don't let OOM or recursive allocation 521 * avoidance (note arenas_cache_bypass check) get in the 522 * way. 523 */ 524 if (ind >= narenas_actual) 525 return (NULL); 526 malloc_mutex_lock(&arenas_lock); 527 arena = arenas[ind]; 528 malloc_mutex_unlock(&arenas_lock); 529 return (arena); 530 } 531 tsd_arenas_cache_set(tsd, arenas_cache); 532 tsd_narenas_cache_set(tsd, narenas_cache); 533 } 534 535 /* 536 * Copy to cache. It's possible that the actual number of arenas has 537 * increased since narenas_total_get() was called above, but that causes 538 * no correctness issues unless two threads concurrently execute the 539 * arenas.extend mallctl, which we trust mallctl synchronization to 540 * prevent. 541 */ 542 malloc_mutex_lock(&arenas_lock); 543 memcpy(arenas_cache, arenas, sizeof(arena_t *) * narenas_actual); 544 malloc_mutex_unlock(&arenas_lock); 545 if (narenas_cache > narenas_actual) { 546 memset(&arenas_cache[narenas_actual], 0, sizeof(arena_t *) * 547 (narenas_cache - narenas_actual)); 548 } 549 550 /* Read the refreshed cache, and init the arena if necessary. */ 551 arena = arenas_cache[ind]; 552 if (init_if_missing && arena == NULL) 553 arena = arenas_cache[ind] = arena_init(ind); 554 return (arena); 555 } 556 557 /* Slow path, called only by arena_choose(). */ 558 arena_t * 559 arena_choose_hard(tsd_t *tsd) 560 { 561 arena_t *ret; 562 563 if (narenas_auto > 1) { 564 unsigned i, choose, first_null; 565 566 choose = 0; 567 first_null = narenas_auto; 568 malloc_mutex_lock(&arenas_lock); 569 assert(a0get() != NULL); 570 for (i = 1; i < narenas_auto; i++) { 571 if (arenas[i] != NULL) { 572 /* 573 * Choose the first arena that has the lowest 574 * number of threads assigned to it. 575 */ 576 if (arenas[i]->nthreads < 577 arenas[choose]->nthreads) 578 choose = i; 579 } else if (first_null == narenas_auto) { 580 /* 581 * Record the index of the first uninitialized 582 * arena, in case all extant arenas are in use. 583 * 584 * NB: It is possible for there to be 585 * discontinuities in terms of initialized 586 * versus uninitialized arenas, due to the 587 * "thread.arena" mallctl. 588 */ 589 first_null = i; 590 } 591 } 592 593 if (arenas[choose]->nthreads == 0 594 || first_null == narenas_auto) { 595 /* 596 * Use an unloaded arena, or the least loaded arena if 597 * all arenas are already initialized. 598 */ 599 ret = arenas[choose]; 600 } else { 601 /* Initialize a new arena. */ 602 choose = first_null; 603 ret = arena_init_locked(choose); 604 if (ret == NULL) { 605 malloc_mutex_unlock(&arenas_lock); 606 return (NULL); 607 } 608 } 609 arena_bind_locked(tsd, choose); 610 malloc_mutex_unlock(&arenas_lock); 611 } else { 612 ret = a0get(); 613 arena_bind(tsd, 0); 614 } 615 616 return (ret); 617 } 618 619 void 620 thread_allocated_cleanup(tsd_t *tsd) 621 { 622 623 /* Do nothing. */ 624 } 625 626 void 627 thread_deallocated_cleanup(tsd_t *tsd) 628 { 629 630 /* Do nothing. */ 631 } 632 633 void 634 arena_cleanup(tsd_t *tsd) 635 { 636 arena_t *arena; 637 638 arena = tsd_arena_get(tsd); 639 if (arena != NULL) 640 arena_unbind(tsd, arena->ind); 641 } 642 643 void 644 arenas_cache_cleanup(tsd_t *tsd) 645 { 646 arena_t **arenas_cache; 647 648 arenas_cache = tsd_arenas_cache_get(tsd); 649 if (arenas_cache != NULL) { 650 /* ANDROID change */ 651 /* Make sure that the arena cache cannot be reused. */ 652 bool *arenas_cache_bypassp = tsd_arenas_cache_bypassp_get(tsd); 653 *arenas_cache_bypassp = true; 654 tsd_arenas_cache_set(tsd, NULL); 655 /* End ANDROID change */ 656 a0dalloc(arenas_cache); 657 } 658 } 659 660 void 661 narenas_cache_cleanup(tsd_t *tsd) 662 { 663 664 /* Do nothing. */ 665 } 666 667 void 668 arenas_cache_bypass_cleanup(tsd_t *tsd) 669 { 670 671 /* Do nothing. */ 672 } 673 674 static void 675 stats_print_atexit(void) 676 { 677 678 if (config_tcache && config_stats) { 679 unsigned narenas, i; 680 681 /* 682 * Merge stats from extant threads. This is racy, since 683 * individual threads do not lock when recording tcache stats 684 * events. As a consequence, the final stats may be slightly 685 * out of date by the time they are reported, if other threads 686 * continue to allocate. 687 */ 688 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 689 arena_t *arena = arenas[i]; 690 if (arena != NULL) { 691 tcache_t *tcache; 692 693 /* 694 * tcache_stats_merge() locks bins, so if any 695 * code is introduced that acquires both arena 696 * and bin locks in the opposite order, 697 * deadlocks may result. 698 */ 699 malloc_mutex_lock(&arena->lock); 700 ql_foreach(tcache, &arena->tcache_ql, link) { 701 tcache_stats_merge(tcache, arena); 702 } 703 malloc_mutex_unlock(&arena->lock); 704 } 705 } 706 } 707 je_malloc_stats_print(NULL, NULL, NULL); 708 } 709 710 /* 711 * End miscellaneous support functions. 712 */ 713 /******************************************************************************/ 714 /* 715 * Begin initialization functions. 716 */ 717 718 #ifndef JEMALLOC_HAVE_SECURE_GETENV 719 static char * 720 secure_getenv(const char *name) 721 { 722 723 # ifdef JEMALLOC_HAVE_ISSETUGID 724 if (issetugid() != 0) 725 return (NULL); 726 # endif 727 return (getenv(name)); 728 } 729 #endif 730 731 static unsigned 732 malloc_ncpus(void) 733 { 734 long result; 735 736 #ifdef _WIN32 737 SYSTEM_INFO si; 738 GetSystemInfo(&si); 739 result = si.dwNumberOfProcessors; 740 #else 741 result = sysconf(_SC_NPROCESSORS_ONLN); 742 #endif 743 return ((result == -1) ? 1 : (unsigned)result); 744 } 745 746 static bool 747 malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, 748 char const **v_p, size_t *vlen_p) 749 { 750 bool accept; 751 const char *opts = *opts_p; 752 753 *k_p = opts; 754 755 for (accept = false; !accept;) { 756 switch (*opts) { 757 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': 758 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': 759 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': 760 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': 761 case 'Y': case 'Z': 762 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': 763 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': 764 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 765 case 's': case 't': case 'u': case 'v': case 'w': case 'x': 766 case 'y': case 'z': 767 case '0': case '1': case '2': case '3': case '4': case '5': 768 case '6': case '7': case '8': case '9': 769 case '_': 770 opts++; 771 break; 772 case ':': 773 opts++; 774 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; 775 *v_p = opts; 776 accept = true; 777 break; 778 case '\0': 779 if (opts != *opts_p) { 780 malloc_write("<jemalloc>: Conf string ends " 781 "with key\n"); 782 } 783 return (true); 784 default: 785 malloc_write("<jemalloc>: Malformed conf string\n"); 786 return (true); 787 } 788 } 789 790 for (accept = false; !accept;) { 791 switch (*opts) { 792 case ',': 793 opts++; 794 /* 795 * Look ahead one character here, because the next time 796 * this function is called, it will assume that end of 797 * input has been cleanly reached if no input remains, 798 * but we have optimistically already consumed the 799 * comma if one exists. 800 */ 801 if (*opts == '\0') { 802 malloc_write("<jemalloc>: Conf string ends " 803 "with comma\n"); 804 } 805 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; 806 accept = true; 807 break; 808 case '\0': 809 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; 810 accept = true; 811 break; 812 default: 813 opts++; 814 break; 815 } 816 } 817 818 *opts_p = opts; 819 return (false); 820 } 821 822 static void 823 malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, 824 size_t vlen) 825 { 826 827 malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k, 828 (int)vlen, v); 829 } 830 831 static void 832 malloc_conf_init(void) 833 { 834 unsigned i; 835 char buf[PATH_MAX + 1]; 836 const char *opts, *k, *v; 837 size_t klen, vlen; 838 839 /* 840 * Automatically configure valgrind before processing options. The 841 * valgrind option remains in jemalloc 3.x for compatibility reasons. 842 */ 843 if (config_valgrind) { 844 in_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false; 845 if (config_fill && unlikely(in_valgrind)) { 846 opt_junk = "false"; 847 opt_junk_alloc = false; 848 opt_junk_free = false; 849 assert(!opt_zero); 850 opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; 851 opt_redzone = true; 852 } 853 if (config_tcache && unlikely(in_valgrind)) 854 opt_tcache = false; 855 } 856 857 #if defined(__ANDROID__) 858 /* Android only supports compiled options. */ 859 for (i = 0; i < 1; i++) { 860 #else 861 for (i = 0; i < 3; i++) { 862 #endif 863 /* Get runtime configuration. */ 864 switch (i) { 865 case 0: 866 if (je_malloc_conf != NULL) { 867 /* 868 * Use options that were compiled into the 869 * program. 870 */ 871 opts = je_malloc_conf; 872 } else { 873 /* No configuration specified. */ 874 buf[0] = '\0'; 875 opts = buf; 876 } 877 break; 878 case 1: { 879 int linklen = 0; 880 #ifndef _WIN32 881 int saved_errno = errno; 882 const char *linkname = 883 # ifdef JEMALLOC_PREFIX 884 "/etc/"JEMALLOC_PREFIX"malloc.conf" 885 # else 886 "/etc/malloc.conf" 887 # endif 888 ; 889 890 /* 891 * Try to use the contents of the "/etc/malloc.conf" 892 * symbolic link's name. 893 */ 894 linklen = readlink(linkname, buf, sizeof(buf) - 1); 895 if (linklen == -1) { 896 /* No configuration specified. */ 897 linklen = 0; 898 /* Restore errno. */ 899 set_errno(saved_errno); 900 } 901 #endif 902 buf[linklen] = '\0'; 903 opts = buf; 904 break; 905 } case 2: { 906 const char *envname = 907 #ifdef JEMALLOC_PREFIX 908 JEMALLOC_CPREFIX"MALLOC_CONF" 909 #else 910 "MALLOC_CONF" 911 #endif 912 ; 913 914 if ((opts = secure_getenv(envname)) != NULL) { 915 /* 916 * Do nothing; opts is already initialized to 917 * the value of the MALLOC_CONF environment 918 * variable. 919 */ 920 } else { 921 /* No configuration specified. */ 922 buf[0] = '\0'; 923 opts = buf; 924 } 925 break; 926 } default: 927 not_reached(); 928 buf[0] = '\0'; 929 opts = buf; 930 } 931 932 while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v, 933 &vlen)) { 934 #define CONF_MATCH(n) \ 935 (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0) 936 #define CONF_MATCH_VALUE(n) \ 937 (sizeof(n)-1 == vlen && strncmp(n, v, vlen) == 0) 938 #define CONF_HANDLE_BOOL(o, n, cont) \ 939 if (CONF_MATCH(n)) { \ 940 if (CONF_MATCH_VALUE("true")) \ 941 o = true; \ 942 else if (CONF_MATCH_VALUE("false")) \ 943 o = false; \ 944 else { \ 945 malloc_conf_error( \ 946 "Invalid conf value", \ 947 k, klen, v, vlen); \ 948 } \ 949 if (cont) \ 950 continue; \ 951 } 952 #define CONF_HANDLE_SIZE_T(o, n, min, max, clip) \ 953 if (CONF_MATCH(n)) { \ 954 uintmax_t um; \ 955 char *end; \ 956 \ 957 set_errno(0); \ 958 um = malloc_strtoumax(v, &end, 0); \ 959 if (get_errno() != 0 || (uintptr_t)end -\ 960 (uintptr_t)v != vlen) { \ 961 malloc_conf_error( \ 962 "Invalid conf value", \ 963 k, klen, v, vlen); \ 964 } else if (clip) { \ 965 if ((min) != 0 && um < (min)) \ 966 o = (min); \ 967 else if (um > (max)) \ 968 o = (max); \ 969 else \ 970 o = um; \ 971 } else { \ 972 if (((min) != 0 && um < (min)) \ 973 || um > (max)) { \ 974 malloc_conf_error( \ 975 "Out-of-range " \ 976 "conf value", \ 977 k, klen, v, vlen); \ 978 } else \ 979 o = um; \ 980 } \ 981 continue; \ 982 } 983 #define CONF_HANDLE_SSIZE_T(o, n, min, max) \ 984 if (CONF_MATCH(n)) { \ 985 long l; \ 986 char *end; \ 987 \ 988 set_errno(0); \ 989 l = strtol(v, &end, 0); \ 990 if (get_errno() != 0 || (uintptr_t)end -\ 991 (uintptr_t)v != vlen) { \ 992 malloc_conf_error( \ 993 "Invalid conf value", \ 994 k, klen, v, vlen); \ 995 } else if (l < (ssize_t)(min) || l > \ 996 (ssize_t)(max)) { \ 997 malloc_conf_error( \ 998 "Out-of-range conf value", \ 999 k, klen, v, vlen); \ 1000 } else \ 1001 o = l; \ 1002 continue; \ 1003 } 1004 #define CONF_HANDLE_CHAR_P(o, n, d) \ 1005 if (CONF_MATCH(n)) { \ 1006 size_t cpylen = (vlen <= \ 1007 sizeof(o)-1) ? vlen : \ 1008 sizeof(o)-1; \ 1009 strncpy(o, v, cpylen); \ 1010 o[cpylen] = '\0'; \ 1011 continue; \ 1012 } 1013 1014 CONF_HANDLE_BOOL(opt_abort, "abort", true) 1015 /* 1016 * Chunks always require at least one header page, 1017 * as many as 2^(LG_SIZE_CLASS_GROUP+1) data pages, and 1018 * possibly an additional page in the presence of 1019 * redzones. In order to simplify options processing, 1020 * use a conservative bound that accommodates all these 1021 * constraints. 1022 */ 1023 CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + 1024 LG_SIZE_CLASS_GROUP + (config_fill ? 2 : 1), 1025 (sizeof(size_t) << 3) - 1, true) 1026 if (strncmp("dss", k, klen) == 0) { 1027 int i; 1028 bool match = false; 1029 for (i = 0; i < dss_prec_limit; i++) { 1030 if (strncmp(dss_prec_names[i], v, vlen) 1031 == 0) { 1032 if (chunk_dss_prec_set(i)) { 1033 malloc_conf_error( 1034 "Error setting dss", 1035 k, klen, v, vlen); 1036 } else { 1037 opt_dss = 1038 dss_prec_names[i]; 1039 match = true; 1040 break; 1041 } 1042 } 1043 } 1044 if (!match) { 1045 malloc_conf_error("Invalid conf value", 1046 k, klen, v, vlen); 1047 } 1048 continue; 1049 } 1050 CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1, 1051 SIZE_T_MAX, false) 1052 CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult", 1053 -1, (sizeof(size_t) << 3) - 1) 1054 CONF_HANDLE_BOOL(opt_stats_print, "stats_print", true) 1055 if (config_fill) { 1056 if (CONF_MATCH("junk")) { 1057 if (CONF_MATCH_VALUE("true")) { 1058 opt_junk = "true"; 1059 opt_junk_alloc = opt_junk_free = 1060 true; 1061 } else if (CONF_MATCH_VALUE("false")) { 1062 opt_junk = "false"; 1063 opt_junk_alloc = opt_junk_free = 1064 false; 1065 } else if (CONF_MATCH_VALUE("alloc")) { 1066 opt_junk = "alloc"; 1067 opt_junk_alloc = true; 1068 opt_junk_free = false; 1069 } else if (CONF_MATCH_VALUE("free")) { 1070 opt_junk = "free"; 1071 opt_junk_alloc = false; 1072 opt_junk_free = true; 1073 } else { 1074 malloc_conf_error( 1075 "Invalid conf value", k, 1076 klen, v, vlen); 1077 } 1078 continue; 1079 } 1080 CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine", 1081 0, SIZE_T_MAX, false) 1082 CONF_HANDLE_BOOL(opt_redzone, "redzone", true) 1083 CONF_HANDLE_BOOL(opt_zero, "zero", true) 1084 } 1085 if (config_utrace) { 1086 CONF_HANDLE_BOOL(opt_utrace, "utrace", true) 1087 } 1088 if (config_xmalloc) { 1089 CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc", true) 1090 } 1091 if (config_tcache) { 1092 CONF_HANDLE_BOOL(opt_tcache, "tcache", 1093 !config_valgrind || !in_valgrind) 1094 if (CONF_MATCH("tcache")) { 1095 assert(config_valgrind && in_valgrind); 1096 if (opt_tcache) { 1097 opt_tcache = false; 1098 malloc_conf_error( 1099 "tcache cannot be enabled " 1100 "while running inside Valgrind", 1101 k, klen, v, vlen); 1102 } 1103 continue; 1104 } 1105 CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, 1106 "lg_tcache_max", -1, 1107 (sizeof(size_t) << 3) - 1) 1108 } 1109 if (config_prof) { 1110 CONF_HANDLE_BOOL(opt_prof, "prof", true) 1111 CONF_HANDLE_CHAR_P(opt_prof_prefix, 1112 "prof_prefix", "jeprof") 1113 CONF_HANDLE_BOOL(opt_prof_active, "prof_active", 1114 true) 1115 CONF_HANDLE_BOOL(opt_prof_thread_active_init, 1116 "prof_thread_active_init", true) 1117 CONF_HANDLE_SIZE_T(opt_lg_prof_sample, 1118 "lg_prof_sample", 0, 1119 (sizeof(uint64_t) << 3) - 1, true) 1120 CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum", 1121 true) 1122 CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, 1123 "lg_prof_interval", -1, 1124 (sizeof(uint64_t) << 3) - 1) 1125 CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump", 1126 true) 1127 CONF_HANDLE_BOOL(opt_prof_final, "prof_final", 1128 true) 1129 CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak", 1130 true) 1131 } 1132 malloc_conf_error("Invalid conf pair", k, klen, v, 1133 vlen); 1134 #undef CONF_MATCH 1135 #undef CONF_HANDLE_BOOL 1136 #undef CONF_HANDLE_SIZE_T 1137 #undef CONF_HANDLE_SSIZE_T 1138 #undef CONF_HANDLE_CHAR_P 1139 } 1140 } 1141 } 1142 1143 /* init_lock must be held. */ 1144 static bool 1145 malloc_init_hard_needed(void) 1146 { 1147 1148 if (malloc_initialized() || (IS_INITIALIZER && malloc_init_state == 1149 malloc_init_recursible)) { 1150 /* 1151 * Another thread initialized the allocator before this one 1152 * acquired init_lock, or this thread is the initializing 1153 * thread, and it is recursively allocating. 1154 */ 1155 return (false); 1156 } 1157 #ifdef JEMALLOC_THREADED_INIT 1158 if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) { 1159 /* Busy-wait until the initializing thread completes. */ 1160 do { 1161 malloc_mutex_unlock(&init_lock); 1162 CPU_SPINWAIT; 1163 malloc_mutex_lock(&init_lock); 1164 } while (!malloc_initialized()); 1165 return (false); 1166 } 1167 #endif 1168 return (true); 1169 } 1170 1171 /* init_lock must be held. */ 1172 static bool 1173 malloc_init_hard_a0_locked(void) 1174 { 1175 1176 malloc_initializer = INITIALIZER; 1177 1178 if (config_prof) 1179 prof_boot0(); 1180 malloc_conf_init(); 1181 if (opt_stats_print) { 1182 /* Print statistics at exit. */ 1183 if (atexit(stats_print_atexit) != 0) { 1184 malloc_write("<jemalloc>: Error in atexit()\n"); 1185 if (opt_abort) 1186 abort(); 1187 } 1188 } 1189 if (base_boot()) 1190 return (true); 1191 if (chunk_boot()) 1192 return (true); 1193 if (ctl_boot()) 1194 return (true); 1195 if (config_prof) 1196 prof_boot1(); 1197 if (arena_boot()) 1198 return (true); 1199 if (config_tcache && tcache_boot()) 1200 return (true); 1201 if (malloc_mutex_init(&arenas_lock)) 1202 return (true); 1203 /* 1204 * Create enough scaffolding to allow recursive allocation in 1205 * malloc_ncpus(). 1206 */ 1207 narenas_total = narenas_auto = 1; 1208 arenas = &a0; 1209 memset(arenas, 0, sizeof(arena_t *) * narenas_auto); 1210 /* 1211 * Initialize one arena here. The rest are lazily created in 1212 * arena_choose_hard(). 1213 */ 1214 if (arena_init(0) == NULL) 1215 return (true); 1216 malloc_init_state = malloc_init_a0_initialized; 1217 return (false); 1218 } 1219 1220 static bool 1221 malloc_init_hard_a0(void) 1222 { 1223 bool ret; 1224 1225 malloc_mutex_lock(&init_lock); 1226 ret = malloc_init_hard_a0_locked(); 1227 malloc_mutex_unlock(&init_lock); 1228 return (ret); 1229 } 1230 1231 /* 1232 * Initialize data structures which may trigger recursive allocation. 1233 * 1234 * init_lock must be held. 1235 */ 1236 static void 1237 malloc_init_hard_recursible(void) 1238 { 1239 1240 malloc_init_state = malloc_init_recursible; 1241 malloc_mutex_unlock(&init_lock); 1242 1243 ncpus = malloc_ncpus(); 1244 1245 #if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \ 1246 && !defined(_WIN32) && !defined(__native_client__)) 1247 /* LinuxThreads's pthread_atfork() allocates. */ 1248 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, 1249 jemalloc_postfork_child) != 0) { 1250 malloc_write("<jemalloc>: Error in pthread_atfork()\n"); 1251 if (opt_abort) 1252 abort(); 1253 } 1254 #endif 1255 malloc_mutex_lock(&init_lock); 1256 } 1257 1258 /* init_lock must be held. */ 1259 static bool 1260 malloc_init_hard_finish(void) 1261 { 1262 1263 if (mutex_boot()) 1264 return (true); 1265 1266 if (opt_narenas == 0) { 1267 /* 1268 * For SMP systems, create more than one arena per CPU by 1269 * default. 1270 */ 1271 if (ncpus > 1) 1272 opt_narenas = ncpus << 2; 1273 else 1274 opt_narenas = 1; 1275 } 1276 #if defined(ANDROID_MAX_ARENAS) 1277 /* Never create more than MAX_ARENAS arenas regardless of num_cpus. 1278 * Extra arenas use more PSS and are not very useful unless 1279 * lots of threads are allocing/freeing at the same time. 1280 */ 1281 if (opt_narenas > ANDROID_MAX_ARENAS) 1282 opt_narenas = ANDROID_MAX_ARENAS; 1283 #endif 1284 narenas_auto = opt_narenas; 1285 /* 1286 * Make sure that the arenas array can be allocated. In practice, this 1287 * limit is enough to allow the allocator to function, but the ctl 1288 * machinery will fail to allocate memory at far lower limits. 1289 */ 1290 if (narenas_auto > chunksize / sizeof(arena_t *)) { 1291 narenas_auto = chunksize / sizeof(arena_t *); 1292 malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n", 1293 narenas_auto); 1294 } 1295 narenas_total = narenas_auto; 1296 1297 /* Allocate and initialize arenas. */ 1298 arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas_total); 1299 if (arenas == NULL) 1300 return (true); 1301 /* 1302 * Zero the array. In practice, this should always be pre-zeroed, 1303 * since it was just mmap()ed, but let's be sure. 1304 */ 1305 memset(arenas, 0, sizeof(arena_t *) * narenas_total); 1306 /* Copy the pointer to the one arena that was already initialized. */ 1307 arenas[0] = a0; 1308 1309 malloc_init_state = malloc_init_initialized; 1310 return (false); 1311 } 1312 1313 static bool 1314 malloc_init_hard(void) 1315 { 1316 1317 malloc_mutex_lock(&init_lock); 1318 if (!malloc_init_hard_needed()) { 1319 malloc_mutex_unlock(&init_lock); 1320 return (false); 1321 } 1322 1323 if (malloc_init_state != malloc_init_a0_initialized && 1324 malloc_init_hard_a0_locked()) { 1325 malloc_mutex_unlock(&init_lock); 1326 return (true); 1327 } 1328 if (malloc_tsd_boot0()) { 1329 malloc_mutex_unlock(&init_lock); 1330 return (true); 1331 } 1332 if (config_prof && prof_boot2()) { 1333 malloc_mutex_unlock(&init_lock); 1334 return (true); 1335 } 1336 1337 malloc_init_hard_recursible(); 1338 1339 if (malloc_init_hard_finish()) { 1340 malloc_mutex_unlock(&init_lock); 1341 return (true); 1342 } 1343 1344 malloc_mutex_unlock(&init_lock); 1345 malloc_tsd_boot1(); 1346 return (false); 1347 } 1348 1349 /* 1350 * End initialization functions. 1351 */ 1352 /******************************************************************************/ 1353 /* 1354 * Begin malloc(3)-compatible functions. 1355 */ 1356 1357 static void * 1358 imalloc_prof_sample(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) 1359 { 1360 void *p; 1361 1362 if (tctx == NULL) 1363 return (NULL); 1364 if (usize <= SMALL_MAXCLASS) { 1365 p = imalloc(tsd, LARGE_MINCLASS); 1366 if (p == NULL) 1367 return (NULL); 1368 arena_prof_promoted(p, usize); 1369 } else 1370 p = imalloc(tsd, usize); 1371 1372 return (p); 1373 } 1374 1375 JEMALLOC_ALWAYS_INLINE_C void * 1376 imalloc_prof(tsd_t *tsd, size_t usize) 1377 { 1378 void *p; 1379 prof_tctx_t *tctx; 1380 1381 tctx = prof_alloc_prep(tsd, usize, true); 1382 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1383 p = imalloc_prof_sample(tsd, usize, tctx); 1384 else 1385 p = imalloc(tsd, usize); 1386 if (unlikely(p == NULL)) { 1387 prof_alloc_rollback(tsd, tctx, true); 1388 return (NULL); 1389 } 1390 prof_malloc(p, usize, tctx); 1391 1392 return (p); 1393 } 1394 1395 JEMALLOC_ALWAYS_INLINE_C void * 1396 imalloc_body(size_t size, tsd_t **tsd, size_t *usize) 1397 { 1398 1399 if (unlikely(malloc_init())) 1400 return (NULL); 1401 *tsd = tsd_fetch(); 1402 1403 if (config_prof && opt_prof) { 1404 *usize = s2u(size); 1405 return (imalloc_prof(*tsd, *usize)); 1406 } 1407 1408 if (config_stats || (config_valgrind && unlikely(in_valgrind))) 1409 *usize = s2u(size); 1410 return (imalloc(*tsd, size)); 1411 } 1412 1413 void * 1414 je_malloc(size_t size) 1415 { 1416 void *ret; 1417 tsd_t *tsd; 1418 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1419 1420 if (size == 0) 1421 size = 1; 1422 1423 ret = imalloc_body(size, &tsd, &usize); 1424 if (unlikely(ret == NULL)) { 1425 if (config_xmalloc && unlikely(opt_xmalloc)) { 1426 malloc_write("<jemalloc>: Error in malloc(): " 1427 "out of memory\n"); 1428 abort(); 1429 } 1430 set_errno(ENOMEM); 1431 } 1432 if (config_stats && likely(ret != NULL)) { 1433 assert(usize == isalloc(ret, config_prof)); 1434 *tsd_thread_allocatedp_get(tsd) += usize; 1435 } 1436 UTRACE(0, size, ret); 1437 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false); 1438 return (ret); 1439 } 1440 1441 static void * 1442 imemalign_prof_sample(tsd_t *tsd, size_t alignment, size_t usize, 1443 prof_tctx_t *tctx) 1444 { 1445 void *p; 1446 1447 if (tctx == NULL) 1448 return (NULL); 1449 if (usize <= SMALL_MAXCLASS) { 1450 assert(sa2u(LARGE_MINCLASS, alignment) == LARGE_MINCLASS); 1451 p = imalloc(tsd, LARGE_MINCLASS); 1452 if (p == NULL) 1453 return (NULL); 1454 arena_prof_promoted(p, usize); 1455 } else 1456 p = ipalloc(tsd, usize, alignment, false); 1457 1458 return (p); 1459 } 1460 1461 JEMALLOC_ALWAYS_INLINE_C void * 1462 imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize) 1463 { 1464 void *p; 1465 prof_tctx_t *tctx; 1466 1467 tctx = prof_alloc_prep(tsd, usize, true); 1468 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1469 p = imemalign_prof_sample(tsd, alignment, usize, tctx); 1470 else 1471 p = ipalloc(tsd, usize, alignment, false); 1472 if (unlikely(p == NULL)) { 1473 prof_alloc_rollback(tsd, tctx, true); 1474 return (NULL); 1475 } 1476 prof_malloc(p, usize, tctx); 1477 1478 return (p); 1479 } 1480 1481 JEMALLOC_ATTR(nonnull(1)) 1482 static int 1483 imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) 1484 { 1485 int ret; 1486 tsd_t *tsd; 1487 size_t usize; 1488 void *result; 1489 1490 assert(min_alignment != 0); 1491 1492 if (unlikely(malloc_init())) { 1493 result = NULL; 1494 goto label_oom; 1495 } else { 1496 tsd = tsd_fetch(); 1497 if (size == 0) 1498 size = 1; 1499 1500 /* Make sure that alignment is a large enough power of 2. */ 1501 if (unlikely(((alignment - 1) & alignment) != 0 1502 || (alignment < min_alignment))) { 1503 if (config_xmalloc && unlikely(opt_xmalloc)) { 1504 malloc_write("<jemalloc>: Error allocating " 1505 "aligned memory: invalid alignment\n"); 1506 abort(); 1507 } 1508 result = NULL; 1509 ret = EINVAL; 1510 goto label_return; 1511 } 1512 1513 usize = sa2u(size, alignment); 1514 if (unlikely(usize == 0)) { 1515 result = NULL; 1516 goto label_oom; 1517 } 1518 1519 if (config_prof && opt_prof) 1520 result = imemalign_prof(tsd, alignment, usize); 1521 else 1522 result = ipalloc(tsd, usize, alignment, false); 1523 if (unlikely(result == NULL)) 1524 goto label_oom; 1525 } 1526 1527 *memptr = result; 1528 ret = 0; 1529 label_return: 1530 if (config_stats && likely(result != NULL)) { 1531 assert(usize == isalloc(result, config_prof)); 1532 *tsd_thread_allocatedp_get(tsd) += usize; 1533 } 1534 UTRACE(0, size, result); 1535 return (ret); 1536 label_oom: 1537 assert(result == NULL); 1538 if (config_xmalloc && unlikely(opt_xmalloc)) { 1539 malloc_write("<jemalloc>: Error allocating aligned memory: " 1540 "out of memory\n"); 1541 abort(); 1542 } 1543 ret = ENOMEM; 1544 goto label_return; 1545 } 1546 1547 int 1548 je_posix_memalign(void **memptr, size_t alignment, size_t size) 1549 { 1550 int ret = imemalign(memptr, alignment, size, sizeof(void *)); 1551 JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr, 1552 config_prof), false); 1553 return (ret); 1554 } 1555 1556 void * 1557 je_aligned_alloc(size_t alignment, size_t size) 1558 { 1559 void *ret; 1560 int err; 1561 1562 if (unlikely((err = imemalign(&ret, alignment, size, 1)) != 0)) { 1563 ret = NULL; 1564 set_errno(err); 1565 } 1566 JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof), 1567 false); 1568 return (ret); 1569 } 1570 1571 static void * 1572 icalloc_prof_sample(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) 1573 { 1574 void *p; 1575 1576 if (tctx == NULL) 1577 return (NULL); 1578 if (usize <= SMALL_MAXCLASS) { 1579 p = icalloc(tsd, LARGE_MINCLASS); 1580 if (p == NULL) 1581 return (NULL); 1582 arena_prof_promoted(p, usize); 1583 } else 1584 p = icalloc(tsd, usize); 1585 1586 return (p); 1587 } 1588 1589 JEMALLOC_ALWAYS_INLINE_C void * 1590 icalloc_prof(tsd_t *tsd, size_t usize) 1591 { 1592 void *p; 1593 prof_tctx_t *tctx; 1594 1595 tctx = prof_alloc_prep(tsd, usize, true); 1596 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1597 p = icalloc_prof_sample(tsd, usize, tctx); 1598 else 1599 p = icalloc(tsd, usize); 1600 if (unlikely(p == NULL)) { 1601 prof_alloc_rollback(tsd, tctx, true); 1602 return (NULL); 1603 } 1604 prof_malloc(p, usize, tctx); 1605 1606 return (p); 1607 } 1608 1609 void * 1610 je_calloc(size_t num, size_t size) 1611 { 1612 void *ret; 1613 tsd_t *tsd; 1614 size_t num_size; 1615 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1616 1617 if (unlikely(malloc_init())) { 1618 num_size = 0; 1619 ret = NULL; 1620 goto label_return; 1621 } 1622 tsd = tsd_fetch(); 1623 1624 num_size = num * size; 1625 if (unlikely(num_size == 0)) { 1626 if (num == 0 || size == 0) 1627 num_size = 1; 1628 else { 1629 ret = NULL; 1630 goto label_return; 1631 } 1632 /* 1633 * Try to avoid division here. We know that it isn't possible to 1634 * overflow during multiplication if neither operand uses any of the 1635 * most significant half of the bits in a size_t. 1636 */ 1637 } else if (unlikely(((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 1638 2))) && (num_size / size != num))) { 1639 /* size_t overflow. */ 1640 ret = NULL; 1641 goto label_return; 1642 } 1643 1644 if (config_prof && opt_prof) { 1645 usize = s2u(num_size); 1646 ret = icalloc_prof(tsd, usize); 1647 } else { 1648 if (config_stats || (config_valgrind && unlikely(in_valgrind))) 1649 usize = s2u(num_size); 1650 ret = icalloc(tsd, num_size); 1651 } 1652 1653 label_return: 1654 if (unlikely(ret == NULL)) { 1655 if (config_xmalloc && unlikely(opt_xmalloc)) { 1656 malloc_write("<jemalloc>: Error in calloc(): out of " 1657 "memory\n"); 1658 abort(); 1659 } 1660 set_errno(ENOMEM); 1661 } 1662 if (config_stats && likely(ret != NULL)) { 1663 assert(usize == isalloc(ret, config_prof)); 1664 *tsd_thread_allocatedp_get(tsd) += usize; 1665 } 1666 UTRACE(0, num_size, ret); 1667 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true); 1668 return (ret); 1669 } 1670 1671 static void * 1672 irealloc_prof_sample(tsd_t *tsd, void *oldptr, size_t old_usize, size_t usize, 1673 prof_tctx_t *tctx) 1674 { 1675 void *p; 1676 1677 if (tctx == NULL) 1678 return (NULL); 1679 if (usize <= SMALL_MAXCLASS) { 1680 p = iralloc(tsd, oldptr, old_usize, LARGE_MINCLASS, 0, false); 1681 if (p == NULL) 1682 return (NULL); 1683 arena_prof_promoted(p, usize); 1684 } else 1685 p = iralloc(tsd, oldptr, old_usize, usize, 0, false); 1686 1687 return (p); 1688 } 1689 1690 JEMALLOC_ALWAYS_INLINE_C void * 1691 irealloc_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t usize) 1692 { 1693 void *p; 1694 prof_tctx_t *old_tctx, *tctx; 1695 1696 old_tctx = prof_tctx_get(oldptr); 1697 tctx = prof_alloc_prep(tsd, usize, true); 1698 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1699 p = irealloc_prof_sample(tsd, oldptr, old_usize, usize, tctx); 1700 else 1701 p = iralloc(tsd, oldptr, old_usize, usize, 0, false); 1702 if (p == NULL) 1703 return (NULL); 1704 prof_realloc(tsd, p, usize, tctx, true, old_usize, old_tctx); 1705 1706 return (p); 1707 } 1708 1709 JEMALLOC_INLINE_C void 1710 ifree(tsd_t *tsd, void *ptr, tcache_t *tcache) 1711 { 1712 size_t usize; 1713 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1714 1715 assert(ptr != NULL); 1716 assert(malloc_initialized() || IS_INITIALIZER); 1717 1718 if (config_prof && opt_prof) { 1719 usize = isalloc(ptr, config_prof); 1720 prof_free(tsd, ptr, usize); 1721 } else if (config_stats || config_valgrind) 1722 usize = isalloc(ptr, config_prof); 1723 if (config_stats) 1724 *tsd_thread_deallocatedp_get(tsd) += usize; 1725 if (config_valgrind && unlikely(in_valgrind)) 1726 rzsize = p2rz(ptr); 1727 iqalloc(tsd, ptr, tcache); 1728 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1729 } 1730 1731 JEMALLOC_INLINE_C void 1732 isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache) 1733 { 1734 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1735 1736 assert(ptr != NULL); 1737 assert(malloc_initialized() || IS_INITIALIZER); 1738 1739 if (config_prof && opt_prof) 1740 prof_free(tsd, ptr, usize); 1741 if (config_stats) 1742 *tsd_thread_deallocatedp_get(tsd) += usize; 1743 if (config_valgrind && unlikely(in_valgrind)) 1744 rzsize = p2rz(ptr); 1745 isqalloc(tsd, ptr, usize, tcache); 1746 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1747 } 1748 1749 void * 1750 je_realloc(void *ptr, size_t size) 1751 { 1752 void *ret; 1753 tsd_t *tsd JEMALLOC_CC_SILENCE_INIT(NULL); 1754 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1755 size_t old_usize = 0; 1756 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1757 1758 if (unlikely(size == 0)) { 1759 if (ptr != NULL) { 1760 /* realloc(ptr, 0) is equivalent to free(ptr). */ 1761 UTRACE(ptr, 0, 0); 1762 tsd = tsd_fetch(); 1763 ifree(tsd, ptr, tcache_get(tsd, false)); 1764 return (NULL); 1765 } 1766 size = 1; 1767 } 1768 1769 if (likely(ptr != NULL)) { 1770 assert(malloc_initialized() || IS_INITIALIZER); 1771 malloc_thread_init(); 1772 tsd = tsd_fetch(); 1773 1774 old_usize = isalloc(ptr, config_prof); 1775 if (config_valgrind && unlikely(in_valgrind)) 1776 old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize); 1777 1778 if (config_prof && opt_prof) { 1779 usize = s2u(size); 1780 ret = irealloc_prof(tsd, ptr, old_usize, usize); 1781 } else { 1782 if (config_stats || (config_valgrind && 1783 unlikely(in_valgrind))) 1784 usize = s2u(size); 1785 ret = iralloc(tsd, ptr, old_usize, size, 0, false); 1786 } 1787 } else { 1788 /* realloc(NULL, size) is equivalent to malloc(size). */ 1789 ret = imalloc_body(size, &tsd, &usize); 1790 } 1791 1792 if (unlikely(ret == NULL)) { 1793 if (config_xmalloc && unlikely(opt_xmalloc)) { 1794 malloc_write("<jemalloc>: Error in realloc(): " 1795 "out of memory\n"); 1796 abort(); 1797 } 1798 set_errno(ENOMEM); 1799 } 1800 if (config_stats && likely(ret != NULL)) { 1801 assert(usize == isalloc(ret, config_prof)); 1802 *tsd_thread_allocatedp_get(tsd) += usize; 1803 *tsd_thread_deallocatedp_get(tsd) += old_usize; 1804 } 1805 UTRACE(ptr, size, ret); 1806 JEMALLOC_VALGRIND_REALLOC(true, ret, usize, true, ptr, old_usize, 1807 old_rzsize, true, false); 1808 return (ret); 1809 } 1810 1811 void 1812 je_free(void *ptr) 1813 { 1814 1815 UTRACE(ptr, 0, 0); 1816 if (likely(ptr != NULL)) { 1817 tsd_t *tsd = tsd_fetch(); 1818 ifree(tsd, ptr, tcache_get(tsd, false)); 1819 } 1820 } 1821 1822 /* 1823 * End malloc(3)-compatible functions. 1824 */ 1825 /******************************************************************************/ 1826 /* 1827 * Begin non-standard override functions. 1828 */ 1829 1830 #ifdef JEMALLOC_OVERRIDE_MEMALIGN 1831 void * 1832 je_memalign(size_t alignment, size_t size) 1833 { 1834 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1835 if (unlikely(imemalign(&ret, alignment, size, 1) != 0)) 1836 ret = NULL; 1837 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); 1838 return (ret); 1839 } 1840 #endif 1841 1842 #ifdef JEMALLOC_OVERRIDE_VALLOC 1843 void * 1844 je_valloc(size_t size) 1845 { 1846 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1847 if (unlikely(imemalign(&ret, PAGE, size, 1) != 0)) 1848 ret = NULL; 1849 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); 1850 return (ret); 1851 } 1852 #endif 1853 1854 /* 1855 * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has 1856 * #define je_malloc malloc 1857 */ 1858 #define malloc_is_malloc 1 1859 #define is_malloc_(a) malloc_is_ ## a 1860 #define is_malloc(a) is_malloc_(a) 1861 1862 #if ((is_malloc(je_malloc) == 1) && defined(JEMALLOC_GLIBC_MALLOC_HOOK)) 1863 /* 1864 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible 1865 * to inconsistently reference libc's malloc(3)-compatible functions 1866 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541). 1867 * 1868 * These definitions interpose hooks in glibc. The functions are actually 1869 * passed an extra argument for the caller return address, which will be 1870 * ignored. 1871 */ 1872 JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free; 1873 JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc; 1874 JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc; 1875 # ifdef JEMALLOC_GLIBC_MEMALIGN_HOOK 1876 JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) = 1877 je_memalign; 1878 # endif 1879 #endif 1880 1881 /* 1882 * End non-standard override functions. 1883 */ 1884 /******************************************************************************/ 1885 /* 1886 * Begin non-standard functions. 1887 */ 1888 1889 JEMALLOC_ALWAYS_INLINE_C bool 1890 imallocx_flags_decode_hard(tsd_t *tsd, size_t size, int flags, size_t *usize, 1891 size_t *alignment, bool *zero, tcache_t **tcache, arena_t **arena) 1892 { 1893 1894 if ((flags & MALLOCX_LG_ALIGN_MASK) == 0) { 1895 *alignment = 0; 1896 *usize = s2u(size); 1897 } else { 1898 *alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags); 1899 *usize = sa2u(size, *alignment); 1900 } 1901 *zero = MALLOCX_ZERO_GET(flags); 1902 if ((flags & MALLOCX_TCACHE_MASK) != 0) { 1903 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 1904 *tcache = NULL; 1905 else 1906 *tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 1907 } else 1908 *tcache = tcache_get(tsd, true); 1909 if ((flags & MALLOCX_ARENA_MASK) != 0) { 1910 unsigned arena_ind = MALLOCX_ARENA_GET(flags); 1911 *arena = arena_get(tsd, arena_ind, true, true); 1912 if (unlikely(*arena == NULL)) 1913 return (true); 1914 } else 1915 *arena = NULL; 1916 return (false); 1917 } 1918 1919 JEMALLOC_ALWAYS_INLINE_C bool 1920 imallocx_flags_decode(tsd_t *tsd, size_t size, int flags, size_t *usize, 1921 size_t *alignment, bool *zero, tcache_t **tcache, arena_t **arena) 1922 { 1923 1924 if (likely(flags == 0)) { 1925 *usize = s2u(size); 1926 assert(usize != 0); 1927 *alignment = 0; 1928 *zero = false; 1929 *tcache = tcache_get(tsd, true); 1930 *arena = NULL; 1931 return (false); 1932 } else { 1933 return (imallocx_flags_decode_hard(tsd, size, flags, usize, 1934 alignment, zero, tcache, arena)); 1935 } 1936 } 1937 1938 JEMALLOC_ALWAYS_INLINE_C void * 1939 imallocx_flags(tsd_t *tsd, size_t usize, size_t alignment, bool zero, 1940 tcache_t *tcache, arena_t *arena) 1941 { 1942 1943 if (alignment != 0) 1944 return (ipalloct(tsd, usize, alignment, zero, tcache, arena)); 1945 if (zero) 1946 return (icalloct(tsd, usize, tcache, arena)); 1947 return (imalloct(tsd, usize, tcache, arena)); 1948 } 1949 1950 JEMALLOC_ALWAYS_INLINE_C void * 1951 imallocx_maybe_flags(tsd_t *tsd, size_t size, int flags, size_t usize, 1952 size_t alignment, bool zero, tcache_t *tcache, arena_t *arena) 1953 { 1954 1955 if (likely(flags == 0)) 1956 return (imalloc(tsd, size)); 1957 return (imallocx_flags(tsd, usize, alignment, zero, tcache, arena)); 1958 } 1959 1960 static void * 1961 imallocx_prof_sample(tsd_t *tsd, size_t size, int flags, size_t usize, 1962 size_t alignment, bool zero, tcache_t *tcache, arena_t *arena) 1963 { 1964 void *p; 1965 1966 if (usize <= SMALL_MAXCLASS) { 1967 assert(((alignment == 0) ? s2u(LARGE_MINCLASS) : 1968 sa2u(LARGE_MINCLASS, alignment)) == LARGE_MINCLASS); 1969 p = imalloct(tsd, LARGE_MINCLASS, tcache, arena); 1970 if (p == NULL) 1971 return (NULL); 1972 arena_prof_promoted(p, usize); 1973 } else { 1974 p = imallocx_maybe_flags(tsd, size, flags, usize, alignment, 1975 zero, tcache, arena); 1976 } 1977 1978 return (p); 1979 } 1980 1981 JEMALLOC_ALWAYS_INLINE_C void * 1982 imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize) 1983 { 1984 void *p; 1985 size_t alignment; 1986 bool zero; 1987 tcache_t *tcache; 1988 arena_t *arena; 1989 prof_tctx_t *tctx; 1990 1991 if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment, 1992 &zero, &tcache, &arena))) 1993 return (NULL); 1994 tctx = prof_alloc_prep(tsd, *usize, true); 1995 if (likely((uintptr_t)tctx == (uintptr_t)1U)) { 1996 p = imallocx_maybe_flags(tsd, size, flags, *usize, alignment, 1997 zero, tcache, arena); 1998 } else if ((uintptr_t)tctx > (uintptr_t)1U) { 1999 p = imallocx_prof_sample(tsd, size, flags, *usize, alignment, 2000 zero, tcache, arena); 2001 } else 2002 p = NULL; 2003 if (unlikely(p == NULL)) { 2004 prof_alloc_rollback(tsd, tctx, true); 2005 return (NULL); 2006 } 2007 prof_malloc(p, *usize, tctx); 2008 2009 return (p); 2010 } 2011 2012 JEMALLOC_ALWAYS_INLINE_C void * 2013 imallocx_no_prof(tsd_t *tsd, size_t size, int flags, size_t *usize) 2014 { 2015 size_t alignment; 2016 bool zero; 2017 tcache_t *tcache; 2018 arena_t *arena; 2019 2020 if (likely(flags == 0)) { 2021 if (config_stats || (config_valgrind && unlikely(in_valgrind))) 2022 *usize = s2u(size); 2023 return (imalloc(tsd, size)); 2024 } 2025 2026 if (unlikely(imallocx_flags_decode_hard(tsd, size, flags, usize, 2027 &alignment, &zero, &tcache, &arena))) 2028 return (NULL); 2029 return (imallocx_flags(tsd, *usize, alignment, zero, tcache, arena)); 2030 } 2031 2032 void * 2033 je_mallocx(size_t size, int flags) 2034 { 2035 tsd_t *tsd; 2036 void *p; 2037 size_t usize; 2038 2039 assert(size != 0); 2040 2041 if (unlikely(malloc_init())) 2042 goto label_oom; 2043 tsd = tsd_fetch(); 2044 2045 if (config_prof && opt_prof) 2046 p = imallocx_prof(tsd, size, flags, &usize); 2047 else 2048 p = imallocx_no_prof(tsd, size, flags, &usize); 2049 if (unlikely(p == NULL)) 2050 goto label_oom; 2051 2052 if (config_stats) { 2053 assert(usize == isalloc(p, config_prof)); 2054 *tsd_thread_allocatedp_get(tsd) += usize; 2055 } 2056 UTRACE(0, size, p); 2057 JEMALLOC_VALGRIND_MALLOC(true, p, usize, MALLOCX_ZERO_GET(flags)); 2058 return (p); 2059 label_oom: 2060 if (config_xmalloc && unlikely(opt_xmalloc)) { 2061 malloc_write("<jemalloc>: Error in mallocx(): out of memory\n"); 2062 abort(); 2063 } 2064 UTRACE(0, size, 0); 2065 return (NULL); 2066 } 2067 2068 static void * 2069 irallocx_prof_sample(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, 2070 size_t alignment, size_t usize, bool zero, tcache_t *tcache, arena_t *arena, 2071 prof_tctx_t *tctx) 2072 { 2073 void *p; 2074 2075 if (tctx == NULL) 2076 return (NULL); 2077 if (usize <= SMALL_MAXCLASS) { 2078 p = iralloct(tsd, oldptr, old_usize, LARGE_MINCLASS, alignment, 2079 zero, tcache, arena); 2080 if (p == NULL) 2081 return (NULL); 2082 arena_prof_promoted(p, usize); 2083 } else { 2084 p = iralloct(tsd, oldptr, old_usize, size, alignment, zero, 2085 tcache, arena); 2086 } 2087 2088 return (p); 2089 } 2090 2091 JEMALLOC_ALWAYS_INLINE_C void * 2092 irallocx_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, 2093 size_t alignment, size_t *usize, bool zero, tcache_t *tcache, 2094 arena_t *arena) 2095 { 2096 void *p; 2097 prof_tctx_t *old_tctx, *tctx; 2098 2099 old_tctx = prof_tctx_get(oldptr); 2100 tctx = prof_alloc_prep(tsd, *usize, false); 2101 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { 2102 p = irallocx_prof_sample(tsd, oldptr, old_usize, size, 2103 alignment, *usize, zero, tcache, arena, tctx); 2104 } else { 2105 p = iralloct(tsd, oldptr, old_usize, size, alignment, zero, 2106 tcache, arena); 2107 } 2108 if (unlikely(p == NULL)) { 2109 prof_alloc_rollback(tsd, tctx, false); 2110 return (NULL); 2111 } 2112 2113 if (p == oldptr && alignment != 0) { 2114 /* 2115 * The allocation did not move, so it is possible that the size 2116 * class is smaller than would guarantee the requested 2117 * alignment, and that the alignment constraint was 2118 * serendipitously satisfied. Additionally, old_usize may not 2119 * be the same as the current usize because of in-place large 2120 * reallocation. Therefore, query the actual value of usize. 2121 */ 2122 *usize = isalloc(p, config_prof); 2123 } 2124 prof_realloc(tsd, p, *usize, tctx, false, old_usize, old_tctx); 2125 2126 return (p); 2127 } 2128 2129 void * 2130 je_rallocx(void *ptr, size_t size, int flags) 2131 { 2132 void *p; 2133 tsd_t *tsd; 2134 size_t usize; 2135 size_t old_usize; 2136 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 2137 size_t alignment = MALLOCX_ALIGN_GET(flags); 2138 bool zero = flags & MALLOCX_ZERO; 2139 arena_t *arena; 2140 tcache_t *tcache; 2141 2142 assert(ptr != NULL); 2143 assert(size != 0); 2144 assert(malloc_initialized() || IS_INITIALIZER); 2145 malloc_thread_init(); 2146 tsd = tsd_fetch(); 2147 2148 if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { 2149 unsigned arena_ind = MALLOCX_ARENA_GET(flags); 2150 arena = arena_get(tsd, arena_ind, true, true); 2151 if (unlikely(arena == NULL)) 2152 goto label_oom; 2153 } else 2154 arena = NULL; 2155 2156 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 2157 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 2158 tcache = NULL; 2159 else 2160 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2161 } else 2162 tcache = tcache_get(tsd, true); 2163 2164 old_usize = isalloc(ptr, config_prof); 2165 if (config_valgrind && unlikely(in_valgrind)) 2166 old_rzsize = u2rz(old_usize); 2167 2168 if (config_prof && opt_prof) { 2169 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 2170 assert(usize != 0); 2171 p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize, 2172 zero, tcache, arena); 2173 if (unlikely(p == NULL)) 2174 goto label_oom; 2175 } else { 2176 p = iralloct(tsd, ptr, old_usize, size, alignment, zero, 2177 tcache, arena); 2178 if (unlikely(p == NULL)) 2179 goto label_oom; 2180 if (config_stats || (config_valgrind && unlikely(in_valgrind))) 2181 usize = isalloc(p, config_prof); 2182 } 2183 2184 if (config_stats) { 2185 *tsd_thread_allocatedp_get(tsd) += usize; 2186 *tsd_thread_deallocatedp_get(tsd) += old_usize; 2187 } 2188 UTRACE(ptr, size, p); 2189 JEMALLOC_VALGRIND_REALLOC(true, p, usize, false, ptr, old_usize, 2190 old_rzsize, false, zero); 2191 return (p); 2192 label_oom: 2193 if (config_xmalloc && unlikely(opt_xmalloc)) { 2194 malloc_write("<jemalloc>: Error in rallocx(): out of memory\n"); 2195 abort(); 2196 } 2197 UTRACE(ptr, size, 0); 2198 return (NULL); 2199 } 2200 2201 JEMALLOC_ALWAYS_INLINE_C size_t 2202 ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra, 2203 size_t alignment, bool zero) 2204 { 2205 size_t usize; 2206 2207 if (ixalloc(ptr, old_usize, size, extra, alignment, zero)) 2208 return (old_usize); 2209 usize = isalloc(ptr, config_prof); 2210 2211 return (usize); 2212 } 2213 2214 static size_t 2215 ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra, 2216 size_t alignment, size_t max_usize, bool zero, prof_tctx_t *tctx) 2217 { 2218 size_t usize; 2219 2220 if (tctx == NULL) 2221 return (old_usize); 2222 /* Use minimum usize to determine whether promotion may happen. */ 2223 if (((alignment == 0) ? s2u(size) : sa2u(size, alignment)) <= 2224 SMALL_MAXCLASS) { 2225 if (ixalloc(ptr, old_usize, SMALL_MAXCLASS+1, 2226 (SMALL_MAXCLASS+1 >= size+extra) ? 0 : size+extra - 2227 (SMALL_MAXCLASS+1), alignment, zero)) 2228 return (old_usize); 2229 usize = isalloc(ptr, config_prof); 2230 if (max_usize < LARGE_MINCLASS) 2231 arena_prof_promoted(ptr, usize); 2232 } else { 2233 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, 2234 zero); 2235 } 2236 2237 return (usize); 2238 } 2239 2240 JEMALLOC_ALWAYS_INLINE_C size_t 2241 ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, 2242 size_t extra, size_t alignment, bool zero) 2243 { 2244 size_t max_usize, usize; 2245 prof_tctx_t *old_tctx, *tctx; 2246 2247 old_tctx = prof_tctx_get(ptr); 2248 /* 2249 * usize isn't knowable before ixalloc() returns when extra is non-zero. 2250 * Therefore, compute its maximum possible value and use that in 2251 * prof_alloc_prep() to decide whether to capture a backtrace. 2252 * prof_realloc() will use the actual usize to decide whether to sample. 2253 */ 2254 max_usize = (alignment == 0) ? s2u(size+extra) : sa2u(size+extra, 2255 alignment); 2256 tctx = prof_alloc_prep(tsd, max_usize, false); 2257 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { 2258 usize = ixallocx_prof_sample(ptr, old_usize, size, extra, 2259 alignment, zero, max_usize, tctx); 2260 } else { 2261 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, 2262 zero); 2263 } 2264 if (unlikely(usize == old_usize)) { 2265 prof_alloc_rollback(tsd, tctx, false); 2266 return (usize); 2267 } 2268 prof_realloc(tsd, ptr, usize, tctx, false, old_usize, old_tctx); 2269 2270 return (usize); 2271 } 2272 2273 size_t 2274 je_xallocx(void *ptr, size_t size, size_t extra, int flags) 2275 { 2276 tsd_t *tsd; 2277 size_t usize, old_usize; 2278 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 2279 size_t alignment = MALLOCX_ALIGN_GET(flags); 2280 bool zero = flags & MALLOCX_ZERO; 2281 2282 assert(ptr != NULL); 2283 assert(size != 0); 2284 assert(SIZE_T_MAX - size >= extra); 2285 assert(malloc_initialized() || IS_INITIALIZER); 2286 malloc_thread_init(); 2287 tsd = tsd_fetch(); 2288 2289 old_usize = isalloc(ptr, config_prof); 2290 if (config_valgrind && unlikely(in_valgrind)) 2291 old_rzsize = u2rz(old_usize); 2292 2293 if (config_prof && opt_prof) { 2294 usize = ixallocx_prof(tsd, ptr, old_usize, size, extra, 2295 alignment, zero); 2296 } else { 2297 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, 2298 zero); 2299 } 2300 if (unlikely(usize == old_usize)) 2301 goto label_not_resized; 2302 2303 if (config_stats) { 2304 *tsd_thread_allocatedp_get(tsd) += usize; 2305 *tsd_thread_deallocatedp_get(tsd) += old_usize; 2306 } 2307 JEMALLOC_VALGRIND_REALLOC(false, ptr, usize, false, ptr, old_usize, 2308 old_rzsize, false, zero); 2309 label_not_resized: 2310 UTRACE(ptr, size, ptr); 2311 return (usize); 2312 } 2313 2314 size_t 2315 je_sallocx(const void *ptr, int flags) 2316 { 2317 size_t usize; 2318 2319 assert(malloc_initialized() || IS_INITIALIZER); 2320 malloc_thread_init(); 2321 2322 if (config_ivsalloc) 2323 usize = ivsalloc(ptr, config_prof); 2324 else 2325 usize = isalloc(ptr, config_prof); 2326 2327 return (usize); 2328 } 2329 2330 void 2331 je_dallocx(void *ptr, int flags) 2332 { 2333 tsd_t *tsd; 2334 tcache_t *tcache; 2335 2336 assert(ptr != NULL); 2337 assert(malloc_initialized() || IS_INITIALIZER); 2338 2339 tsd = tsd_fetch(); 2340 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 2341 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 2342 tcache = NULL; 2343 else 2344 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2345 } else 2346 tcache = tcache_get(tsd, false); 2347 2348 UTRACE(ptr, 0, 0); 2349 ifree(tsd_fetch(), ptr, tcache); 2350 } 2351 2352 JEMALLOC_ALWAYS_INLINE_C size_t 2353 inallocx(size_t size, int flags) 2354 { 2355 size_t usize; 2356 2357 if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) 2358 usize = s2u(size); 2359 else 2360 usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags)); 2361 assert(usize != 0); 2362 return (usize); 2363 } 2364 2365 void 2366 je_sdallocx(void *ptr, size_t size, int flags) 2367 { 2368 tsd_t *tsd; 2369 tcache_t *tcache; 2370 size_t usize; 2371 2372 assert(ptr != NULL); 2373 assert(malloc_initialized() || IS_INITIALIZER); 2374 usize = inallocx(size, flags); 2375 assert(usize == isalloc(ptr, config_prof)); 2376 2377 tsd = tsd_fetch(); 2378 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 2379 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 2380 tcache = NULL; 2381 else 2382 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2383 } else 2384 tcache = tcache_get(tsd, false); 2385 2386 UTRACE(ptr, 0, 0); 2387 isfree(tsd, ptr, usize, tcache); 2388 } 2389 2390 size_t 2391 je_nallocx(size_t size, int flags) 2392 { 2393 2394 assert(size != 0); 2395 2396 if (unlikely(malloc_init())) 2397 return (0); 2398 2399 return (inallocx(size, flags)); 2400 } 2401 2402 int 2403 je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, 2404 size_t newlen) 2405 { 2406 2407 if (unlikely(malloc_init())) 2408 return (EAGAIN); 2409 2410 return (ctl_byname(name, oldp, oldlenp, newp, newlen)); 2411 } 2412 2413 int 2414 je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) 2415 { 2416 2417 if (unlikely(malloc_init())) 2418 return (EAGAIN); 2419 2420 return (ctl_nametomib(name, mibp, miblenp)); 2421 } 2422 2423 int 2424 je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 2425 void *newp, size_t newlen) 2426 { 2427 2428 if (unlikely(malloc_init())) 2429 return (EAGAIN); 2430 2431 return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); 2432 } 2433 2434 void 2435 je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, 2436 const char *opts) 2437 { 2438 2439 stats_print(write_cb, cbopaque, opts); 2440 } 2441 2442 size_t 2443 je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) 2444 { 2445 size_t ret; 2446 2447 assert(malloc_initialized() || IS_INITIALIZER); 2448 malloc_thread_init(); 2449 2450 if (config_ivsalloc) 2451 ret = ivsalloc(ptr, config_prof); 2452 else 2453 ret = (ptr == NULL) ? 0 : isalloc(ptr, config_prof); 2454 2455 return (ret); 2456 } 2457 2458 /* 2459 * End non-standard functions. 2460 */ 2461 /******************************************************************************/ 2462 /* 2463 * The following functions are used by threading libraries for protection of 2464 * malloc during fork(). 2465 */ 2466 2467 /* 2468 * If an application creates a thread before doing any allocation in the main 2469 * thread, then calls fork(2) in the main thread followed by memory allocation 2470 * in the child process, a race can occur that results in deadlock within the 2471 * child: the main thread may have forked while the created thread had 2472 * partially initialized the allocator. Ordinarily jemalloc prevents 2473 * fork/malloc races via the following functions it registers during 2474 * initialization using pthread_atfork(), but of course that does no good if 2475 * the allocator isn't fully initialized at fork time. The following library 2476 * constructor is a partial solution to this problem. It may still be possible 2477 * to trigger the deadlock described above, but doing so would involve forking 2478 * via a library constructor that runs before jemalloc's runs. 2479 */ 2480 JEMALLOC_ATTR(constructor) 2481 static void 2482 jemalloc_constructor(void) 2483 { 2484 2485 malloc_init(); 2486 } 2487 2488 #ifndef JEMALLOC_MUTEX_INIT_CB 2489 void 2490 jemalloc_prefork(void) 2491 #else 2492 JEMALLOC_EXPORT void 2493 _malloc_prefork(void) 2494 #endif 2495 { 2496 unsigned i; 2497 2498 #ifdef JEMALLOC_MUTEX_INIT_CB 2499 if (!malloc_initialized()) 2500 return; 2501 #endif 2502 assert(malloc_initialized()); 2503 2504 /* Acquire all mutexes in a safe order. */ 2505 ctl_prefork(); 2506 prof_prefork(); 2507 malloc_mutex_prefork(&arenas_lock); 2508 for (i = 0; i < narenas_total; i++) { 2509 if (arenas[i] != NULL) 2510 arena_prefork(arenas[i]); 2511 } 2512 chunk_prefork(); 2513 base_prefork(); 2514 } 2515 2516 #ifndef JEMALLOC_MUTEX_INIT_CB 2517 void 2518 jemalloc_postfork_parent(void) 2519 #else 2520 JEMALLOC_EXPORT void 2521 _malloc_postfork(void) 2522 #endif 2523 { 2524 unsigned i; 2525 2526 #ifdef JEMALLOC_MUTEX_INIT_CB 2527 if (!malloc_initialized()) 2528 return; 2529 #endif 2530 assert(malloc_initialized()); 2531 2532 /* Release all mutexes, now that fork() has completed. */ 2533 base_postfork_parent(); 2534 chunk_postfork_parent(); 2535 for (i = 0; i < narenas_total; i++) { 2536 if (arenas[i] != NULL) 2537 arena_postfork_parent(arenas[i]); 2538 } 2539 malloc_mutex_postfork_parent(&arenas_lock); 2540 prof_postfork_parent(); 2541 ctl_postfork_parent(); 2542 } 2543 2544 void 2545 jemalloc_postfork_child(void) 2546 { 2547 unsigned i; 2548 2549 assert(malloc_initialized()); 2550 2551 /* Release all mutexes, now that fork() has completed. */ 2552 base_postfork_child(); 2553 chunk_postfork_child(); 2554 for (i = 0; i < narenas_total; i++) { 2555 if (arenas[i] != NULL) 2556 arena_postfork_child(arenas[i]); 2557 } 2558 malloc_mutex_postfork_child(&arenas_lock); 2559 prof_postfork_child(); 2560 ctl_postfork_child(); 2561 } 2562 2563 /******************************************************************************/ 2564 2565 /* ANDROID change */ 2566 /* This is an implementation that uses the same arena access pattern found 2567 * in the arena_stats_merge function from src/arena.c. 2568 */ 2569 struct mallinfo je_mallinfo() { 2570 struct mallinfo mi; 2571 memset(&mi, 0, sizeof(mi)); 2572 2573 malloc_mutex_lock(&arenas_lock); 2574 for (unsigned i = 0; i < narenas_auto; i++) { 2575 if (arenas[i] != NULL) { 2576 malloc_mutex_lock(&arenas[i]->lock); 2577 mi.hblkhd += arenas[i]->stats.mapped; 2578 mi.uordblks += arenas[i]->stats.allocated_large; 2579 mi.uordblks += arenas[i]->stats.allocated_huge; 2580 malloc_mutex_unlock(&arenas[i]->lock); 2581 2582 for (unsigned j = 0; j < NBINS; j++) { 2583 arena_bin_t* bin = &arenas[i]->bins[j]; 2584 2585 malloc_mutex_lock(&bin->lock); 2586 mi.uordblks += arena_bin_info[j].reg_size * bin->stats.curregs; 2587 malloc_mutex_unlock(&bin->lock); 2588 } 2589 } 2590 } 2591 malloc_mutex_unlock(&arenas_lock); 2592 mi.fordblks = mi.hblkhd - mi.uordblks; 2593 mi.usmblks = mi.hblkhd; 2594 return mi; 2595 } 2596 2597 size_t __mallinfo_narenas() { 2598 return narenas_auto; 2599 } 2600 2601 size_t __mallinfo_nbins() { 2602 return NBINS; 2603 } 2604 2605 struct mallinfo __mallinfo_arena_info(size_t aidx) { 2606 struct mallinfo mi; 2607 memset(&mi, 0, sizeof(mi)); 2608 2609 malloc_mutex_lock(&arenas_lock); 2610 if (aidx < narenas_auto) { 2611 if (arenas[aidx] != NULL) { 2612 malloc_mutex_lock(&arenas[aidx]->lock); 2613 mi.hblkhd = arenas[aidx]->stats.mapped; 2614 mi.ordblks = arenas[aidx]->stats.allocated_large; 2615 mi.uordblks = arenas[aidx]->stats.allocated_huge; 2616 malloc_mutex_unlock(&arenas[aidx]->lock); 2617 2618 for (unsigned j = 0; j < NBINS; j++) { 2619 arena_bin_t* bin = &arenas[aidx]->bins[j]; 2620 2621 malloc_mutex_lock(&bin->lock); 2622 mi.fsmblks += arena_bin_info[j].reg_size * bin->stats.curregs; 2623 malloc_mutex_unlock(&bin->lock); 2624 } 2625 } 2626 } 2627 malloc_mutex_unlock(&arenas_lock); 2628 return mi; 2629 } 2630 2631 struct mallinfo __mallinfo_bin_info(size_t aidx, size_t bidx) { 2632 struct mallinfo mi; 2633 memset(&mi, 0, sizeof(mi)); 2634 2635 malloc_mutex_lock(&arenas_lock); 2636 if (aidx < narenas_auto && bidx < NBINS) { 2637 if (arenas[aidx] != NULL) { 2638 arena_bin_t* bin = &arenas[aidx]->bins[bidx]; 2639 2640 malloc_mutex_lock(&bin->lock); 2641 mi.ordblks = arena_bin_info[bidx].reg_size * bin->stats.curregs; 2642 mi.uordblks = bin->stats.nmalloc; 2643 mi.fordblks = bin->stats.ndalloc; 2644 malloc_mutex_unlock(&bin->lock); 2645 } 2646 } 2647 malloc_mutex_unlock(&arenas_lock); 2648 return mi; 2649 } 2650 /* End ANDROID change */ 2651
