1 #define JEMALLOC_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 4 /******************************************************************************/ 5 /* Data. */ 6 7 malloc_tsd_data(, arenas, arena_t *, NULL) 8 malloc_tsd_data(, thread_allocated, thread_allocated_t, 9 THREAD_ALLOCATED_INITIALIZER) 10 11 /* Runtime configuration options. */ 12 const char *je_malloc_conf; 13 bool opt_abort = 14 #ifdef JEMALLOC_DEBUG 15 true 16 #else 17 false 18 #endif 19 ; 20 bool opt_junk = 21 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 22 true 23 #else 24 false 25 #endif 26 ; 27 size_t opt_quarantine = ZU(0); 28 bool opt_redzone = false; 29 bool opt_utrace = false; 30 bool opt_xmalloc = false; 31 bool opt_zero = false; 32 size_t opt_narenas = 0; 33 34 /* Initialized to true if the process is running inside Valgrind. */ 35 bool in_valgrind; 36 37 unsigned ncpus; 38 39 malloc_mutex_t arenas_lock; 40 arena_t **arenas; 41 unsigned narenas_total; 42 unsigned narenas_auto; 43 44 /* Set to true once the allocator has been initialized. */ 45 static bool malloc_initialized = false; 46 47 #ifdef JEMALLOC_THREADED_INIT 48 /* Used to let the initializing thread recursively allocate. */ 49 # define NO_INITIALIZER ((unsigned long)0) 50 # define INITIALIZER pthread_self() 51 # define IS_INITIALIZER (malloc_initializer == pthread_self()) 52 static pthread_t malloc_initializer = NO_INITIALIZER; 53 #else 54 # define NO_INITIALIZER false 55 # define INITIALIZER true 56 # define IS_INITIALIZER malloc_initializer 57 static bool malloc_initializer = NO_INITIALIZER; 58 #endif 59 60 /* Used to avoid initialization races. */ 61 #ifdef _WIN32 62 static malloc_mutex_t init_lock; 63 64 JEMALLOC_ATTR(constructor) 65 static void WINAPI 66 _init_init_lock(void) 67 { 68 69 malloc_mutex_init(&init_lock); 70 } 71 72 #ifdef _MSC_VER 73 # pragma section(".CRT$XCU", read) 74 JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used) 75 static const void (WINAPI *init_init_lock)(void) = _init_init_lock; 76 #endif 77 78 #else 79 static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; 80 #endif 81 82 typedef struct { 83 void *p; /* Input pointer (as in realloc(p, s)). */ 84 size_t s; /* Request size. */ 85 void *r; /* Result pointer. */ 86 } malloc_utrace_t; 87 88 #ifdef JEMALLOC_UTRACE 89 # define UTRACE(a, b, c) do { \ 90 if (opt_utrace) { \ 91 int utrace_serrno = errno; \ 92 malloc_utrace_t ut; \ 93 ut.p = (a); \ 94 ut.s = (b); \ 95 ut.r = (c); \ 96 utrace(&ut, sizeof(ut)); \ 97 errno = utrace_serrno; \ 98 } \ 99 } while (0) 100 #else 101 # define UTRACE(a, b, c) 102 #endif 103 104 /******************************************************************************/ 105 /* 106 * Function prototypes for static functions that are referenced prior to 107 * definition. 108 */ 109 110 static bool malloc_init_hard(void); 111 112 /******************************************************************************/ 113 /* 114 * Begin miscellaneous support functions. 115 */ 116 117 /* Create a new arena and insert it into the arenas array at index ind. */ 118 arena_t * 119 arenas_extend(unsigned ind) 120 { 121 arena_t *ret; 122 123 ret = (arena_t *)base_alloc(sizeof(arena_t)); 124 if (ret != NULL && arena_new(ret, ind) == false) { 125 arenas[ind] = ret; 126 return (ret); 127 } 128 /* Only reached if there is an OOM error. */ 129 130 /* 131 * OOM here is quite inconvenient to propagate, since dealing with it 132 * would require a check for failure in the fast path. Instead, punt 133 * by using arenas[0]. In practice, this is an extremely unlikely 134 * failure. 135 */ 136 malloc_write("<jemalloc>: Error initializing arena\n"); 137 if (opt_abort) 138 abort(); 139 140 return (arenas[0]); 141 } 142 143 /* Slow path, called only by choose_arena(). */ 144 arena_t * 145 choose_arena_hard(void) 146 { 147 arena_t *ret; 148 149 if (narenas_auto > 1) { 150 unsigned i, choose, first_null; 151 152 choose = 0; 153 first_null = narenas_auto; 154 malloc_mutex_lock(&arenas_lock); 155 assert(arenas[0] != NULL); 156 for (i = 1; i < narenas_auto; i++) { 157 if (arenas[i] != NULL) { 158 /* 159 * Choose the first arena that has the lowest 160 * number of threads assigned to it. 161 */ 162 if (arenas[i]->nthreads < 163 arenas[choose]->nthreads) 164 choose = i; 165 } else if (first_null == narenas_auto) { 166 /* 167 * Record the index of the first uninitialized 168 * arena, in case all extant arenas are in use. 169 * 170 * NB: It is possible for there to be 171 * discontinuities in terms of initialized 172 * versus uninitialized arenas, due to the 173 * "thread.arena" mallctl. 174 */ 175 first_null = i; 176 } 177 } 178 179 if (arenas[choose]->nthreads == 0 180 || first_null == narenas_auto) { 181 /* 182 * Use an unloaded arena, or the least loaded arena if 183 * all arenas are already initialized. 184 */ 185 ret = arenas[choose]; 186 } else { 187 /* Initialize a new arena. */ 188 ret = arenas_extend(first_null); 189 } 190 ret->nthreads++; 191 malloc_mutex_unlock(&arenas_lock); 192 } else { 193 ret = arenas[0]; 194 malloc_mutex_lock(&arenas_lock); 195 ret->nthreads++; 196 malloc_mutex_unlock(&arenas_lock); 197 } 198 199 arenas_tsd_set(&ret); 200 201 return (ret); 202 } 203 204 static void 205 stats_print_atexit(void) 206 { 207 208 if (config_tcache && config_stats) { 209 unsigned narenas, i; 210 211 /* 212 * Merge stats from extant threads. This is racy, since 213 * individual threads do not lock when recording tcache stats 214 * events. As a consequence, the final stats may be slightly 215 * out of date by the time they are reported, if other threads 216 * continue to allocate. 217 */ 218 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 219 arena_t *arena = arenas[i]; 220 if (arena != NULL) { 221 tcache_t *tcache; 222 223 /* 224 * tcache_stats_merge() locks bins, so if any 225 * code is introduced that acquires both arena 226 * and bin locks in the opposite order, 227 * deadlocks may result. 228 */ 229 malloc_mutex_lock(&arena->lock); 230 ql_foreach(tcache, &arena->tcache_ql, link) { 231 tcache_stats_merge(tcache, arena); 232 } 233 malloc_mutex_unlock(&arena->lock); 234 } 235 } 236 } 237 je_malloc_stats_print(NULL, NULL, NULL); 238 } 239 240 /* 241 * End miscellaneous support functions. 242 */ 243 /******************************************************************************/ 244 /* 245 * Begin initialization functions. 246 */ 247 248 static unsigned 249 malloc_ncpus(void) 250 { 251 long result; 252 253 #ifdef _WIN32 254 SYSTEM_INFO si; 255 GetSystemInfo(&si); 256 result = si.dwNumberOfProcessors; 257 #else 258 result = sysconf(_SC_NPROCESSORS_ONLN); 259 #endif 260 return ((result == -1) ? 1 : (unsigned)result); 261 } 262 263 void 264 arenas_cleanup(void *arg) 265 { 266 arena_t *arena = *(arena_t **)arg; 267 268 malloc_mutex_lock(&arenas_lock); 269 arena->nthreads--; 270 malloc_mutex_unlock(&arenas_lock); 271 } 272 273 JEMALLOC_ALWAYS_INLINE_C void 274 malloc_thread_init(void) 275 { 276 277 /* 278 * TSD initialization can't be safely done as a side effect of 279 * deallocation, because it is possible for a thread to do nothing but 280 * deallocate its TLS data via free(), in which case writing to TLS 281 * would cause write-after-free memory corruption. The quarantine 282 * facility *only* gets used as a side effect of deallocation, so make 283 * a best effort attempt at initializing its TSD by hooking all 284 * allocation events. 285 */ 286 if (config_fill && opt_quarantine) 287 quarantine_alloc_hook(); 288 } 289 290 JEMALLOC_ALWAYS_INLINE_C bool 291 malloc_init(void) 292 { 293 294 if (malloc_initialized == false && malloc_init_hard()) 295 return (true); 296 malloc_thread_init(); 297 298 return (false); 299 } 300 301 static bool 302 malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, 303 char const **v_p, size_t *vlen_p) 304 { 305 bool accept; 306 const char *opts = *opts_p; 307 308 *k_p = opts; 309 310 for (accept = false; accept == false;) { 311 switch (*opts) { 312 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': 313 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': 314 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': 315 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': 316 case 'Y': case 'Z': 317 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': 318 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': 319 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 320 case 's': case 't': case 'u': case 'v': case 'w': case 'x': 321 case 'y': case 'z': 322 case '0': case '1': case '2': case '3': case '4': case '5': 323 case '6': case '7': case '8': case '9': 324 case '_': 325 opts++; 326 break; 327 case ':': 328 opts++; 329 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; 330 *v_p = opts; 331 accept = true; 332 break; 333 case '\0': 334 if (opts != *opts_p) { 335 malloc_write("<jemalloc>: Conf string ends " 336 "with key\n"); 337 } 338 return (true); 339 default: 340 malloc_write("<jemalloc>: Malformed conf string\n"); 341 return (true); 342 } 343 } 344 345 for (accept = false; accept == false;) { 346 switch (*opts) { 347 case ',': 348 opts++; 349 /* 350 * Look ahead one character here, because the next time 351 * this function is called, it will assume that end of 352 * input has been cleanly reached if no input remains, 353 * but we have optimistically already consumed the 354 * comma if one exists. 355 */ 356 if (*opts == '\0') { 357 malloc_write("<jemalloc>: Conf string ends " 358 "with comma\n"); 359 } 360 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; 361 accept = true; 362 break; 363 case '\0': 364 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; 365 accept = true; 366 break; 367 default: 368 opts++; 369 break; 370 } 371 } 372 373 *opts_p = opts; 374 return (false); 375 } 376 377 static void 378 malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, 379 size_t vlen) 380 { 381 382 malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k, 383 (int)vlen, v); 384 } 385 386 static void 387 malloc_conf_init(void) 388 { 389 unsigned i; 390 char buf[PATH_MAX + 1]; 391 const char *opts, *k, *v; 392 size_t klen, vlen; 393 394 /* 395 * Automatically configure valgrind before processing options. The 396 * valgrind option remains in jemalloc 3.x for compatibility reasons. 397 */ 398 if (config_valgrind) { 399 in_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false; 400 if (config_fill && in_valgrind) { 401 opt_junk = false; 402 assert(opt_zero == false); 403 opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; 404 opt_redzone = true; 405 } 406 if (config_tcache && in_valgrind) 407 opt_tcache = false; 408 } 409 410 #if defined(__ANDROID__) 411 /* Android only supports compiled options. */ 412 for (i = 0; i < 1; i++) { 413 #else 414 for (i = 0; i < 3; i++) { 415 #endif 416 /* Get runtime configuration. */ 417 switch (i) { 418 case 0: 419 if (je_malloc_conf != NULL) { 420 /* 421 * Use options that were compiled into the 422 * program. 423 */ 424 opts = je_malloc_conf; 425 } else { 426 /* No configuration specified. */ 427 buf[0] = '\0'; 428 opts = buf; 429 } 430 break; 431 case 1: { 432 int linklen = 0; 433 #ifndef _WIN32 434 int saved_errno = errno; 435 const char *linkname = 436 # ifdef JEMALLOC_PREFIX 437 "/etc/"JEMALLOC_PREFIX"malloc.conf" 438 # else 439 "/etc/malloc.conf" 440 # endif 441 ; 442 443 /* 444 * Try to use the contents of the "/etc/malloc.conf" 445 * symbolic link's name. 446 */ 447 linklen = readlink(linkname, buf, sizeof(buf) - 1); 448 if (linklen == -1) { 449 /* No configuration specified. */ 450 linklen = 0; 451 /* restore errno */ 452 set_errno(saved_errno); 453 } 454 #endif 455 buf[linklen] = '\0'; 456 opts = buf; 457 break; 458 } case 2: { 459 const char *envname = 460 #ifdef JEMALLOC_PREFIX 461 JEMALLOC_CPREFIX"MALLOC_CONF" 462 #else 463 "MALLOC_CONF" 464 #endif 465 ; 466 467 if ((opts = getenv(envname)) != NULL) { 468 /* 469 * Do nothing; opts is already initialized to 470 * the value of the MALLOC_CONF environment 471 * variable. 472 */ 473 } else { 474 /* No configuration specified. */ 475 buf[0] = '\0'; 476 opts = buf; 477 } 478 break; 479 } default: 480 not_reached(); 481 buf[0] = '\0'; 482 opts = buf; 483 } 484 485 while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v, 486 &vlen) == false) { 487 #define CONF_MATCH(n) \ 488 (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0) 489 #define CONF_HANDLE_BOOL(o, n, cont) \ 490 if (CONF_MATCH(n)) { \ 491 if (strncmp("true", v, vlen) == 0 && \ 492 vlen == sizeof("true")-1) \ 493 o = true; \ 494 else if (strncmp("false", v, vlen) == \ 495 0 && vlen == sizeof("false")-1) \ 496 o = false; \ 497 else { \ 498 malloc_conf_error( \ 499 "Invalid conf value", \ 500 k, klen, v, vlen); \ 501 } \ 502 if (cont) \ 503 continue; \ 504 } 505 #define CONF_HANDLE_SIZE_T(o, n, min, max, clip) \ 506 if (CONF_MATCH(n)) { \ 507 uintmax_t um; \ 508 char *end; \ 509 \ 510 set_errno(0); \ 511 um = malloc_strtoumax(v, &end, 0); \ 512 if (get_errno() != 0 || (uintptr_t)end -\ 513 (uintptr_t)v != vlen) { \ 514 malloc_conf_error( \ 515 "Invalid conf value", \ 516 k, klen, v, vlen); \ 517 } else if (clip) { \ 518 if (min != 0 && um < min) \ 519 o = min; \ 520 else if (um > max) \ 521 o = max; \ 522 else \ 523 o = um; \ 524 } else { \ 525 if ((min != 0 && um < min) || \ 526 um > max) { \ 527 malloc_conf_error( \ 528 "Out-of-range " \ 529 "conf value", \ 530 k, klen, v, vlen); \ 531 } else \ 532 o = um; \ 533 } \ 534 continue; \ 535 } 536 #define CONF_HANDLE_SSIZE_T(o, n, min, max) \ 537 if (CONF_MATCH(n)) { \ 538 long l; \ 539 char *end; \ 540 \ 541 set_errno(0); \ 542 l = strtol(v, &end, 0); \ 543 if (get_errno() != 0 || (uintptr_t)end -\ 544 (uintptr_t)v != vlen) { \ 545 malloc_conf_error( \ 546 "Invalid conf value", \ 547 k, klen, v, vlen); \ 548 } else if (l < (ssize_t)min || l > \ 549 (ssize_t)max) { \ 550 malloc_conf_error( \ 551 "Out-of-range conf value", \ 552 k, klen, v, vlen); \ 553 } else \ 554 o = l; \ 555 continue; \ 556 } 557 #define CONF_HANDLE_CHAR_P(o, n, d) \ 558 if (CONF_MATCH(n)) { \ 559 size_t cpylen = (vlen <= \ 560 sizeof(o)-1) ? vlen : \ 561 sizeof(o)-1; \ 562 strncpy(o, v, cpylen); \ 563 o[cpylen] = '\0'; \ 564 continue; \ 565 } 566 567 CONF_HANDLE_BOOL(opt_abort, "abort", true) 568 /* 569 * Chunks always require at least one header page, plus 570 * one data page in the absence of redzones, or three 571 * pages in the presence of redzones. In order to 572 * simplify options processing, fix the limit based on 573 * config_fill. 574 */ 575 CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + 576 (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1, 577 true) 578 if (strncmp("dss", k, klen) == 0) { 579 int i; 580 bool match = false; 581 for (i = 0; i < dss_prec_limit; i++) { 582 if (strncmp(dss_prec_names[i], v, vlen) 583 == 0) { 584 if (chunk_dss_prec_set(i)) { 585 malloc_conf_error( 586 "Error setting dss", 587 k, klen, v, vlen); 588 } else { 589 opt_dss = 590 dss_prec_names[i]; 591 match = true; 592 break; 593 } 594 } 595 } 596 if (match == false) { 597 malloc_conf_error("Invalid conf value", 598 k, klen, v, vlen); 599 } 600 continue; 601 } 602 CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1, 603 SIZE_T_MAX, false) 604 CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult", 605 -1, (sizeof(size_t) << 3) - 1) 606 CONF_HANDLE_BOOL(opt_stats_print, "stats_print", true) 607 if (config_fill) { 608 CONF_HANDLE_BOOL(opt_junk, "junk", true) 609 CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine", 610 0, SIZE_T_MAX, false) 611 CONF_HANDLE_BOOL(opt_redzone, "redzone", true) 612 CONF_HANDLE_BOOL(opt_zero, "zero", true) 613 } 614 if (config_utrace) { 615 CONF_HANDLE_BOOL(opt_utrace, "utrace", true) 616 } 617 if (config_xmalloc) { 618 CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc", true) 619 } 620 if (config_tcache) { 621 CONF_HANDLE_BOOL(opt_tcache, "tcache", 622 !config_valgrind || !in_valgrind) 623 if (CONF_MATCH("tcache")) { 624 assert(config_valgrind && in_valgrind); 625 if (opt_tcache) { 626 opt_tcache = false; 627 malloc_conf_error( 628 "tcache cannot be enabled " 629 "while running inside Valgrind", 630 k, klen, v, vlen); 631 } 632 continue; 633 } 634 CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, 635 "lg_tcache_max", -1, 636 (sizeof(size_t) << 3) - 1) 637 } 638 if (config_prof) { 639 CONF_HANDLE_BOOL(opt_prof, "prof", true) 640 CONF_HANDLE_CHAR_P(opt_prof_prefix, 641 "prof_prefix", "jeprof") 642 CONF_HANDLE_BOOL(opt_prof_active, "prof_active", 643 true) 644 CONF_HANDLE_SSIZE_T(opt_lg_prof_sample, 645 "lg_prof_sample", 0, 646 (sizeof(uint64_t) << 3) - 1) 647 CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum", 648 true) 649 CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, 650 "lg_prof_interval", -1, 651 (sizeof(uint64_t) << 3) - 1) 652 CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump", 653 true) 654 CONF_HANDLE_BOOL(opt_prof_final, "prof_final", 655 true) 656 CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak", 657 true) 658 } 659 malloc_conf_error("Invalid conf pair", k, klen, v, 660 vlen); 661 #undef CONF_MATCH 662 #undef CONF_HANDLE_BOOL 663 #undef CONF_HANDLE_SIZE_T 664 #undef CONF_HANDLE_SSIZE_T 665 #undef CONF_HANDLE_CHAR_P 666 } 667 } 668 } 669 670 static bool 671 malloc_init_hard(void) 672 { 673 arena_t *init_arenas[1]; 674 675 malloc_mutex_lock(&init_lock); 676 if (malloc_initialized || IS_INITIALIZER) { 677 /* 678 * Another thread initialized the allocator before this one 679 * acquired init_lock, or this thread is the initializing 680 * thread, and it is recursively allocating. 681 */ 682 malloc_mutex_unlock(&init_lock); 683 return (false); 684 } 685 #ifdef JEMALLOC_THREADED_INIT 686 if (malloc_initializer != NO_INITIALIZER && IS_INITIALIZER == false) { 687 /* Busy-wait until the initializing thread completes. */ 688 do { 689 malloc_mutex_unlock(&init_lock); 690 CPU_SPINWAIT; 691 malloc_mutex_lock(&init_lock); 692 } while (malloc_initialized == false); 693 malloc_mutex_unlock(&init_lock); 694 return (false); 695 } 696 #endif 697 malloc_initializer = INITIALIZER; 698 699 malloc_tsd_boot(); 700 if (config_prof) 701 prof_boot0(); 702 703 malloc_conf_init(); 704 705 if (opt_stats_print) { 706 /* Print statistics at exit. */ 707 if (atexit(stats_print_atexit) != 0) { 708 malloc_write("<jemalloc>: Error in atexit()\n"); 709 if (opt_abort) 710 abort(); 711 } 712 } 713 714 if (base_boot()) { 715 malloc_mutex_unlock(&init_lock); 716 return (true); 717 } 718 719 if (chunk_boot()) { 720 malloc_mutex_unlock(&init_lock); 721 return (true); 722 } 723 724 if (ctl_boot()) { 725 malloc_mutex_unlock(&init_lock); 726 return (true); 727 } 728 729 if (config_prof) 730 prof_boot1(); 731 732 arena_boot(); 733 734 if (config_tcache && tcache_boot0()) { 735 malloc_mutex_unlock(&init_lock); 736 return (true); 737 } 738 739 if (huge_boot()) { 740 malloc_mutex_unlock(&init_lock); 741 return (true); 742 } 743 744 if (malloc_mutex_init(&arenas_lock)) { 745 malloc_mutex_unlock(&init_lock); 746 return (true); 747 } 748 749 /* 750 * Create enough scaffolding to allow recursive allocation in 751 * malloc_ncpus(). 752 */ 753 narenas_total = narenas_auto = 1; 754 arenas = init_arenas; 755 memset(arenas, 0, sizeof(arena_t *) * narenas_auto); 756 757 /* 758 * Initialize one arena here. The rest are lazily created in 759 * choose_arena_hard(). 760 */ 761 arenas_extend(0); 762 if (arenas[0] == NULL) { 763 malloc_mutex_unlock(&init_lock); 764 return (true); 765 } 766 767 /* Initialize allocation counters before any allocations can occur. */ 768 if (config_stats && thread_allocated_tsd_boot()) { 769 malloc_mutex_unlock(&init_lock); 770 return (true); 771 } 772 773 if (arenas_tsd_boot()) { 774 malloc_mutex_unlock(&init_lock); 775 return (true); 776 } 777 778 if (config_tcache && tcache_boot1()) { 779 malloc_mutex_unlock(&init_lock); 780 return (true); 781 } 782 783 if (config_fill && quarantine_boot()) { 784 malloc_mutex_unlock(&init_lock); 785 return (true); 786 } 787 788 if (config_prof && prof_boot2()) { 789 malloc_mutex_unlock(&init_lock); 790 return (true); 791 } 792 793 malloc_mutex_unlock(&init_lock); 794 /**********************************************************************/ 795 /* Recursive allocation may follow. */ 796 797 ncpus = malloc_ncpus(); 798 799 #if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \ 800 && !defined(_WIN32) && !defined(__native_client__)) 801 /* LinuxThreads's pthread_atfork() allocates. */ 802 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, 803 jemalloc_postfork_child) != 0) { 804 malloc_write("<jemalloc>: Error in pthread_atfork()\n"); 805 if (opt_abort) 806 abort(); 807 } 808 #endif 809 810 /* Done recursively allocating. */ 811 /**********************************************************************/ 812 malloc_mutex_lock(&init_lock); 813 814 if (mutex_boot()) { 815 malloc_mutex_unlock(&init_lock); 816 return (true); 817 } 818 819 if (opt_narenas == 0) { 820 /* 821 * For SMP systems, create more than one arena per CPU by 822 * default. 823 */ 824 if (ncpus > 1) 825 opt_narenas = ncpus << 2; 826 else 827 opt_narenas = 1; 828 } 829 #if defined(ANDROID_MAX_ARENAS) 830 /* Never create more than MAX_ARENAS arenas regardless of num_cpus. 831 * Extra arenas use more PSS and are not very useful unless 832 * lots of threads are allocing/freeing at the same time. 833 */ 834 if (opt_narenas > ANDROID_MAX_ARENAS) 835 opt_narenas = ANDROID_MAX_ARENAS; 836 #endif 837 narenas_auto = opt_narenas; 838 /* 839 * Make sure that the arenas array can be allocated. In practice, this 840 * limit is enough to allow the allocator to function, but the ctl 841 * machinery will fail to allocate memory at far lower limits. 842 */ 843 if (narenas_auto > chunksize / sizeof(arena_t *)) { 844 narenas_auto = chunksize / sizeof(arena_t *); 845 malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n", 846 narenas_auto); 847 } 848 narenas_total = narenas_auto; 849 850 /* Allocate and initialize arenas. */ 851 arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas_total); 852 if (arenas == NULL) { 853 malloc_mutex_unlock(&init_lock); 854 return (true); 855 } 856 /* 857 * Zero the array. In practice, this should always be pre-zeroed, 858 * since it was just mmap()ed, but let's be sure. 859 */ 860 memset(arenas, 0, sizeof(arena_t *) * narenas_total); 861 /* Copy the pointer to the one arena that was already initialized. */ 862 arenas[0] = init_arenas[0]; 863 864 malloc_initialized = true; 865 malloc_mutex_unlock(&init_lock); 866 867 return (false); 868 } 869 870 /* 871 * End initialization functions. 872 */ 873 /******************************************************************************/ 874 /* 875 * Begin malloc(3)-compatible functions. 876 */ 877 878 static void * 879 imalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt) 880 { 881 void *p; 882 883 if (cnt == NULL) 884 return (NULL); 885 if (usize <= SMALL_MAXCLASS) { 886 p = imalloc(SMALL_MAXCLASS+1); 887 if (p == NULL) 888 return (NULL); 889 arena_prof_promoted(p, usize); 890 } else 891 p = imalloc(usize); 892 893 return (p); 894 } 895 896 JEMALLOC_ALWAYS_INLINE_C void * 897 imalloc_prof(size_t usize) 898 { 899 void *p; 900 prof_thr_cnt_t *cnt; 901 902 PROF_ALLOC_PREP(usize, cnt); 903 if ((uintptr_t)cnt != (uintptr_t)1U) 904 p = imalloc_prof_sample(usize, cnt); 905 else 906 p = imalloc(usize); 907 if (p == NULL) 908 return (NULL); 909 prof_malloc(p, usize, cnt); 910 911 return (p); 912 } 913 914 JEMALLOC_ALWAYS_INLINE_C void * 915 imalloc_body(size_t size, size_t *usize) 916 { 917 918 if (malloc_init()) 919 return (NULL); 920 921 if (config_prof && opt_prof) { 922 *usize = s2u(size); 923 return (imalloc_prof(*usize)); 924 } 925 926 if (config_stats || (config_valgrind && in_valgrind)) 927 *usize = s2u(size); 928 return (imalloc(size)); 929 } 930 931 void * 932 je_malloc(size_t size) 933 { 934 void *ret; 935 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 936 937 if (size == 0) 938 size = 1; 939 940 ret = imalloc_body(size, &usize); 941 if (ret == NULL) { 942 if (config_xmalloc && opt_xmalloc) { 943 malloc_write("<jemalloc>: Error in malloc(): " 944 "out of memory\n"); 945 abort(); 946 } 947 set_errno(ENOMEM); 948 } 949 if (config_stats && ret != NULL) { 950 assert(usize == isalloc(ret, config_prof)); 951 thread_allocated_tsd_get()->allocated += usize; 952 } 953 UTRACE(0, size, ret); 954 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false); 955 return (ret); 956 } 957 958 static void * 959 imemalign_prof_sample(size_t alignment, size_t usize, prof_thr_cnt_t *cnt) 960 { 961 void *p; 962 963 if (cnt == NULL) 964 return (NULL); 965 if (usize <= SMALL_MAXCLASS) { 966 assert(sa2u(SMALL_MAXCLASS+1, alignment) != 0); 967 p = ipalloc(sa2u(SMALL_MAXCLASS+1, alignment), alignment, 968 false); 969 if (p == NULL) 970 return (NULL); 971 arena_prof_promoted(p, usize); 972 } else 973 p = ipalloc(usize, alignment, false); 974 975 return (p); 976 } 977 978 JEMALLOC_ALWAYS_INLINE_C void * 979 imemalign_prof(size_t alignment, size_t usize, prof_thr_cnt_t *cnt) 980 { 981 void *p; 982 983 if ((uintptr_t)cnt != (uintptr_t)1U) 984 p = imemalign_prof_sample(alignment, usize, cnt); 985 else 986 p = ipalloc(usize, alignment, false); 987 if (p == NULL) 988 return (NULL); 989 prof_malloc(p, usize, cnt); 990 991 return (p); 992 } 993 994 JEMALLOC_ATTR(nonnull(1)) 995 static int 996 imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) 997 { 998 int ret; 999 size_t usize; 1000 void *result; 1001 1002 assert(min_alignment != 0); 1003 1004 if (malloc_init()) { 1005 result = NULL; 1006 goto label_oom; 1007 } else { 1008 if (size == 0) 1009 size = 1; 1010 1011 /* Make sure that alignment is a large enough power of 2. */ 1012 if (((alignment - 1) & alignment) != 0 1013 || (alignment < min_alignment)) { 1014 if (config_xmalloc && opt_xmalloc) { 1015 malloc_write("<jemalloc>: Error allocating " 1016 "aligned memory: invalid alignment\n"); 1017 abort(); 1018 } 1019 result = NULL; 1020 ret = EINVAL; 1021 goto label_return; 1022 } 1023 1024 usize = sa2u(size, alignment); 1025 if (usize == 0) { 1026 result = NULL; 1027 goto label_oom; 1028 } 1029 1030 if (config_prof && opt_prof) { 1031 prof_thr_cnt_t *cnt; 1032 1033 PROF_ALLOC_PREP(usize, cnt); 1034 result = imemalign_prof(alignment, usize, cnt); 1035 } else 1036 result = ipalloc(usize, alignment, false); 1037 if (result == NULL) 1038 goto label_oom; 1039 } 1040 1041 *memptr = result; 1042 ret = 0; 1043 label_return: 1044 if (config_stats && result != NULL) { 1045 assert(usize == isalloc(result, config_prof)); 1046 thread_allocated_tsd_get()->allocated += usize; 1047 } 1048 UTRACE(0, size, result); 1049 return (ret); 1050 label_oom: 1051 assert(result == NULL); 1052 if (config_xmalloc && opt_xmalloc) { 1053 malloc_write("<jemalloc>: Error allocating aligned memory: " 1054 "out of memory\n"); 1055 abort(); 1056 } 1057 ret = ENOMEM; 1058 goto label_return; 1059 } 1060 1061 int 1062 je_posix_memalign(void **memptr, size_t alignment, size_t size) 1063 { 1064 int ret = imemalign(memptr, alignment, size, sizeof(void *)); 1065 JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr, 1066 config_prof), false); 1067 return (ret); 1068 } 1069 1070 void * 1071 je_aligned_alloc(size_t alignment, size_t size) 1072 { 1073 void *ret; 1074 int err; 1075 1076 if ((err = imemalign(&ret, alignment, size, 1)) != 0) { 1077 ret = NULL; 1078 set_errno(err); 1079 } 1080 JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof), 1081 false); 1082 return (ret); 1083 } 1084 1085 static void * 1086 icalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt) 1087 { 1088 void *p; 1089 1090 if (cnt == NULL) 1091 return (NULL); 1092 if (usize <= SMALL_MAXCLASS) { 1093 p = icalloc(SMALL_MAXCLASS+1); 1094 if (p == NULL) 1095 return (NULL); 1096 arena_prof_promoted(p, usize); 1097 } else 1098 p = icalloc(usize); 1099 1100 return (p); 1101 } 1102 1103 JEMALLOC_ALWAYS_INLINE_C void * 1104 icalloc_prof(size_t usize, prof_thr_cnt_t *cnt) 1105 { 1106 void *p; 1107 1108 if ((uintptr_t)cnt != (uintptr_t)1U) 1109 p = icalloc_prof_sample(usize, cnt); 1110 else 1111 p = icalloc(usize); 1112 if (p == NULL) 1113 return (NULL); 1114 prof_malloc(p, usize, cnt); 1115 1116 return (p); 1117 } 1118 1119 void * 1120 je_calloc(size_t num, size_t size) 1121 { 1122 void *ret; 1123 size_t num_size; 1124 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1125 1126 if (malloc_init()) { 1127 num_size = 0; 1128 ret = NULL; 1129 goto label_return; 1130 } 1131 1132 num_size = num * size; 1133 if (num_size == 0) { 1134 if (num == 0 || size == 0) 1135 num_size = 1; 1136 else { 1137 ret = NULL; 1138 goto label_return; 1139 } 1140 /* 1141 * Try to avoid division here. We know that it isn't possible to 1142 * overflow during multiplication if neither operand uses any of the 1143 * most significant half of the bits in a size_t. 1144 */ 1145 } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2))) 1146 && (num_size / size != num)) { 1147 /* size_t overflow. */ 1148 ret = NULL; 1149 goto label_return; 1150 } 1151 1152 if (config_prof && opt_prof) { 1153 prof_thr_cnt_t *cnt; 1154 1155 usize = s2u(num_size); 1156 PROF_ALLOC_PREP(usize, cnt); 1157 ret = icalloc_prof(usize, cnt); 1158 } else { 1159 if (config_stats || (config_valgrind && in_valgrind)) 1160 usize = s2u(num_size); 1161 ret = icalloc(num_size); 1162 } 1163 1164 label_return: 1165 if (ret == NULL) { 1166 if (config_xmalloc && opt_xmalloc) { 1167 malloc_write("<jemalloc>: Error in calloc(): out of " 1168 "memory\n"); 1169 abort(); 1170 } 1171 set_errno(ENOMEM); 1172 } 1173 if (config_stats && ret != NULL) { 1174 assert(usize == isalloc(ret, config_prof)); 1175 thread_allocated_tsd_get()->allocated += usize; 1176 } 1177 UTRACE(0, num_size, ret); 1178 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true); 1179 return (ret); 1180 } 1181 1182 static void * 1183 irealloc_prof_sample(void *oldptr, size_t usize, prof_thr_cnt_t *cnt) 1184 { 1185 void *p; 1186 1187 if (cnt == NULL) 1188 return (NULL); 1189 if (usize <= SMALL_MAXCLASS) { 1190 p = iralloc(oldptr, SMALL_MAXCLASS+1, 0, 0, false); 1191 if (p == NULL) 1192 return (NULL); 1193 arena_prof_promoted(p, usize); 1194 } else 1195 p = iralloc(oldptr, usize, 0, 0, false); 1196 1197 return (p); 1198 } 1199 1200 JEMALLOC_ALWAYS_INLINE_C void * 1201 irealloc_prof(void *oldptr, size_t old_usize, size_t usize, prof_thr_cnt_t *cnt) 1202 { 1203 void *p; 1204 prof_ctx_t *old_ctx; 1205 1206 old_ctx = prof_ctx_get(oldptr); 1207 if ((uintptr_t)cnt != (uintptr_t)1U) 1208 p = irealloc_prof_sample(oldptr, usize, cnt); 1209 else 1210 p = iralloc(oldptr, usize, 0, 0, false); 1211 if (p == NULL) 1212 return (NULL); 1213 prof_realloc(p, usize, cnt, old_usize, old_ctx); 1214 1215 return (p); 1216 } 1217 1218 JEMALLOC_INLINE_C void 1219 ifree(void *ptr) 1220 { 1221 size_t usize; 1222 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1223 1224 assert(ptr != NULL); 1225 assert(malloc_initialized || IS_INITIALIZER); 1226 1227 if (config_prof && opt_prof) { 1228 usize = isalloc(ptr, config_prof); 1229 prof_free(ptr, usize); 1230 } else if (config_stats || config_valgrind) 1231 usize = isalloc(ptr, config_prof); 1232 if (config_stats) 1233 thread_allocated_tsd_get()->deallocated += usize; 1234 if (config_valgrind && in_valgrind) 1235 rzsize = p2rz(ptr); 1236 iqalloc(ptr); 1237 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1238 } 1239 1240 void * 1241 je_realloc(void *ptr, size_t size) 1242 { 1243 void *ret; 1244 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1245 size_t old_usize = 0; 1246 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1247 1248 if (size == 0) { 1249 if (ptr != NULL) { 1250 /* realloc(ptr, 0) is equivalent to free(ptr). */ 1251 UTRACE(ptr, 0, 0); 1252 ifree(ptr); 1253 return (NULL); 1254 } 1255 size = 1; 1256 } 1257 1258 if (ptr != NULL) { 1259 assert(malloc_initialized || IS_INITIALIZER); 1260 malloc_thread_init(); 1261 1262 if ((config_prof && opt_prof) || config_stats || 1263 (config_valgrind && in_valgrind)) 1264 old_usize = isalloc(ptr, config_prof); 1265 if (config_valgrind && in_valgrind) 1266 old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize); 1267 1268 if (config_prof && opt_prof) { 1269 prof_thr_cnt_t *cnt; 1270 1271 usize = s2u(size); 1272 PROF_ALLOC_PREP(usize, cnt); 1273 ret = irealloc_prof(ptr, old_usize, usize, cnt); 1274 } else { 1275 if (config_stats || (config_valgrind && in_valgrind)) 1276 usize = s2u(size); 1277 ret = iralloc(ptr, size, 0, 0, false); 1278 } 1279 } else { 1280 /* realloc(NULL, size) is equivalent to malloc(size). */ 1281 ret = imalloc_body(size, &usize); 1282 } 1283 1284 if (ret == NULL) { 1285 if (config_xmalloc && opt_xmalloc) { 1286 malloc_write("<jemalloc>: Error in realloc(): " 1287 "out of memory\n"); 1288 abort(); 1289 } 1290 set_errno(ENOMEM); 1291 } 1292 if (config_stats && ret != NULL) { 1293 thread_allocated_t *ta; 1294 assert(usize == isalloc(ret, config_prof)); 1295 ta = thread_allocated_tsd_get(); 1296 ta->allocated += usize; 1297 ta->deallocated += old_usize; 1298 } 1299 UTRACE(ptr, size, ret); 1300 JEMALLOC_VALGRIND_REALLOC(true, ret, usize, true, ptr, old_usize, 1301 old_rzsize, true, false); 1302 return (ret); 1303 } 1304 1305 void 1306 je_free(void *ptr) 1307 { 1308 1309 UTRACE(ptr, 0, 0); 1310 if (ptr != NULL) 1311 ifree(ptr); 1312 } 1313 1314 /* 1315 * End malloc(3)-compatible functions. 1316 */ 1317 /******************************************************************************/ 1318 /* 1319 * Begin non-standard override functions. 1320 */ 1321 1322 #ifdef JEMALLOC_OVERRIDE_MEMALIGN 1323 void * 1324 je_memalign(size_t alignment, size_t size) 1325 { 1326 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1327 imemalign(&ret, alignment, size, 1); 1328 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); 1329 return (ret); 1330 } 1331 #endif 1332 1333 #ifdef JEMALLOC_OVERRIDE_VALLOC 1334 void * 1335 je_valloc(size_t size) 1336 { 1337 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1338 imemalign(&ret, PAGE, size, 1); 1339 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); 1340 return (ret); 1341 } 1342 #endif 1343 1344 /* 1345 * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has 1346 * #define je_malloc malloc 1347 */ 1348 #define malloc_is_malloc 1 1349 #define is_malloc_(a) malloc_is_ ## a 1350 #define is_malloc(a) is_malloc_(a) 1351 1352 #if ((is_malloc(je_malloc) == 1) && defined(__GLIBC__) && !defined(__UCLIBC__)) 1353 /* 1354 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible 1355 * to inconsistently reference libc's malloc(3)-compatible functions 1356 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541). 1357 * 1358 * These definitions interpose hooks in glibc. The functions are actually 1359 * passed an extra argument for the caller return address, which will be 1360 * ignored. 1361 */ 1362 JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free; 1363 JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc; 1364 JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc; 1365 JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) = 1366 je_memalign; 1367 #endif 1368 1369 /* 1370 * End non-standard override functions. 1371 */ 1372 /******************************************************************************/ 1373 /* 1374 * Begin non-standard functions. 1375 */ 1376 1377 JEMALLOC_ALWAYS_INLINE_C void * 1378 imallocx(size_t usize, size_t alignment, bool zero, bool try_tcache, 1379 arena_t *arena) 1380 { 1381 1382 assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, 1383 alignment))); 1384 1385 if (alignment != 0) 1386 return (ipalloct(usize, alignment, zero, try_tcache, arena)); 1387 else if (zero) 1388 return (icalloct(usize, try_tcache, arena)); 1389 else 1390 return (imalloct(usize, try_tcache, arena)); 1391 } 1392 1393 static void * 1394 imallocx_prof_sample(size_t usize, size_t alignment, bool zero, bool try_tcache, 1395 arena_t *arena, prof_thr_cnt_t *cnt) 1396 { 1397 void *p; 1398 1399 if (cnt == NULL) 1400 return (NULL); 1401 if (usize <= SMALL_MAXCLASS) { 1402 size_t usize_promoted = (alignment == 0) ? 1403 s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1, alignment); 1404 assert(usize_promoted != 0); 1405 p = imallocx(usize_promoted, alignment, zero, try_tcache, 1406 arena); 1407 if (p == NULL) 1408 return (NULL); 1409 arena_prof_promoted(p, usize); 1410 } else 1411 p = imallocx(usize, alignment, zero, try_tcache, arena); 1412 1413 return (p); 1414 } 1415 1416 JEMALLOC_ALWAYS_INLINE_C void * 1417 imallocx_prof(size_t usize, size_t alignment, bool zero, bool try_tcache, 1418 arena_t *arena, prof_thr_cnt_t *cnt) 1419 { 1420 void *p; 1421 1422 if ((uintptr_t)cnt != (uintptr_t)1U) { 1423 p = imallocx_prof_sample(usize, alignment, zero, try_tcache, 1424 arena, cnt); 1425 } else 1426 p = imallocx(usize, alignment, zero, try_tcache, arena); 1427 if (p == NULL) 1428 return (NULL); 1429 prof_malloc(p, usize, cnt); 1430 1431 return (p); 1432 } 1433 1434 void * 1435 je_mallocx(size_t size, int flags) 1436 { 1437 void *p; 1438 size_t usize; 1439 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) 1440 & (SIZE_T_MAX-1)); 1441 bool zero = flags & MALLOCX_ZERO; 1442 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; 1443 arena_t *arena; 1444 bool try_tcache; 1445 1446 assert(size != 0); 1447 1448 if (malloc_init()) 1449 goto label_oom; 1450 1451 if (arena_ind != UINT_MAX) { 1452 arena = arenas[arena_ind]; 1453 try_tcache = false; 1454 } else { 1455 arena = NULL; 1456 try_tcache = true; 1457 } 1458 1459 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 1460 assert(usize != 0); 1461 1462 if (config_prof && opt_prof) { 1463 prof_thr_cnt_t *cnt; 1464 1465 PROF_ALLOC_PREP(usize, cnt); 1466 p = imallocx_prof(usize, alignment, zero, try_tcache, arena, 1467 cnt); 1468 } else 1469 p = imallocx(usize, alignment, zero, try_tcache, arena); 1470 if (p == NULL) 1471 goto label_oom; 1472 1473 if (config_stats) { 1474 assert(usize == isalloc(p, config_prof)); 1475 thread_allocated_tsd_get()->allocated += usize; 1476 } 1477 UTRACE(0, size, p); 1478 JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero); 1479 return (p); 1480 label_oom: 1481 if (config_xmalloc && opt_xmalloc) { 1482 malloc_write("<jemalloc>: Error in mallocx(): out of memory\n"); 1483 abort(); 1484 } 1485 UTRACE(0, size, 0); 1486 return (NULL); 1487 } 1488 1489 static void * 1490 irallocx_prof_sample(void *oldptr, size_t size, size_t alignment, size_t usize, 1491 bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena, 1492 prof_thr_cnt_t *cnt) 1493 { 1494 void *p; 1495 1496 if (cnt == NULL) 1497 return (NULL); 1498 if (usize <= SMALL_MAXCLASS) { 1499 p = iralloct(oldptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= 1500 size) ? 0 : size - (SMALL_MAXCLASS+1), alignment, zero, 1501 try_tcache_alloc, try_tcache_dalloc, arena); 1502 if (p == NULL) 1503 return (NULL); 1504 arena_prof_promoted(p, usize); 1505 } else { 1506 p = iralloct(oldptr, size, 0, alignment, zero, 1507 try_tcache_alloc, try_tcache_dalloc, arena); 1508 } 1509 1510 return (p); 1511 } 1512 1513 JEMALLOC_ALWAYS_INLINE_C void * 1514 irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment, 1515 size_t *usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, 1516 arena_t *arena, prof_thr_cnt_t *cnt) 1517 { 1518 void *p; 1519 prof_ctx_t *old_ctx; 1520 1521 old_ctx = prof_ctx_get(oldptr); 1522 if ((uintptr_t)cnt != (uintptr_t)1U) 1523 p = irallocx_prof_sample(oldptr, size, alignment, *usize, zero, 1524 try_tcache_alloc, try_tcache_dalloc, arena, cnt); 1525 else { 1526 p = iralloct(oldptr, size, 0, alignment, zero, 1527 try_tcache_alloc, try_tcache_dalloc, arena); 1528 } 1529 if (p == NULL) 1530 return (NULL); 1531 1532 if (p == oldptr && alignment != 0) { 1533 /* 1534 * The allocation did not move, so it is possible that the size 1535 * class is smaller than would guarantee the requested 1536 * alignment, and that the alignment constraint was 1537 * serendipitously satisfied. Additionally, old_usize may not 1538 * be the same as the current usize because of in-place large 1539 * reallocation. Therefore, query the actual value of usize. 1540 */ 1541 *usize = isalloc(p, config_prof); 1542 } 1543 prof_realloc(p, *usize, cnt, old_usize, old_ctx); 1544 1545 return (p); 1546 } 1547 1548 void * 1549 je_rallocx(void *ptr, size_t size, int flags) 1550 { 1551 void *p; 1552 size_t usize, old_usize; 1553 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1554 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) 1555 & (SIZE_T_MAX-1)); 1556 bool zero = flags & MALLOCX_ZERO; 1557 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; 1558 bool try_tcache_alloc, try_tcache_dalloc; 1559 arena_t *arena; 1560 1561 assert(ptr != NULL); 1562 assert(size != 0); 1563 assert(malloc_initialized || IS_INITIALIZER); 1564 malloc_thread_init(); 1565 1566 if (arena_ind != UINT_MAX) { 1567 arena_chunk_t *chunk; 1568 try_tcache_alloc = false; 1569 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1570 try_tcache_dalloc = (chunk == ptr || chunk->arena != 1571 arenas[arena_ind]); 1572 arena = arenas[arena_ind]; 1573 } else { 1574 try_tcache_alloc = true; 1575 try_tcache_dalloc = true; 1576 arena = NULL; 1577 } 1578 1579 if ((config_prof && opt_prof) || config_stats || 1580 (config_valgrind && in_valgrind)) 1581 old_usize = isalloc(ptr, config_prof); 1582 if (config_valgrind && in_valgrind) 1583 old_rzsize = u2rz(old_usize); 1584 1585 if (config_prof && opt_prof) { 1586 prof_thr_cnt_t *cnt; 1587 1588 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 1589 assert(usize != 0); 1590 PROF_ALLOC_PREP(usize, cnt); 1591 p = irallocx_prof(ptr, old_usize, size, alignment, &usize, zero, 1592 try_tcache_alloc, try_tcache_dalloc, arena, cnt); 1593 if (p == NULL) 1594 goto label_oom; 1595 } else { 1596 p = iralloct(ptr, size, 0, alignment, zero, try_tcache_alloc, 1597 try_tcache_dalloc, arena); 1598 if (p == NULL) 1599 goto label_oom; 1600 if (config_stats || (config_valgrind && in_valgrind)) 1601 usize = isalloc(p, config_prof); 1602 } 1603 1604 if (config_stats) { 1605 thread_allocated_t *ta; 1606 ta = thread_allocated_tsd_get(); 1607 ta->allocated += usize; 1608 ta->deallocated += old_usize; 1609 } 1610 UTRACE(ptr, size, p); 1611 JEMALLOC_VALGRIND_REALLOC(true, p, usize, false, ptr, old_usize, 1612 old_rzsize, false, zero); 1613 return (p); 1614 label_oom: 1615 if (config_xmalloc && opt_xmalloc) { 1616 malloc_write("<jemalloc>: Error in rallocx(): out of memory\n"); 1617 abort(); 1618 } 1619 UTRACE(ptr, size, 0); 1620 return (NULL); 1621 } 1622 1623 JEMALLOC_ALWAYS_INLINE_C size_t 1624 ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra, 1625 size_t alignment, bool zero, arena_t *arena) 1626 { 1627 size_t usize; 1628 1629 if (ixalloc(ptr, size, extra, alignment, zero)) 1630 return (old_usize); 1631 usize = isalloc(ptr, config_prof); 1632 1633 return (usize); 1634 } 1635 1636 static size_t 1637 ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra, 1638 size_t alignment, size_t max_usize, bool zero, arena_t *arena, 1639 prof_thr_cnt_t *cnt) 1640 { 1641 size_t usize; 1642 1643 if (cnt == NULL) 1644 return (old_usize); 1645 /* Use minimum usize to determine whether promotion may happen. */ 1646 if (((alignment == 0) ? s2u(size) : sa2u(size, alignment)) <= 1647 SMALL_MAXCLASS) { 1648 if (ixalloc(ptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= 1649 size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1), 1650 alignment, zero)) 1651 return (old_usize); 1652 usize = isalloc(ptr, config_prof); 1653 if (max_usize < PAGE) 1654 arena_prof_promoted(ptr, usize); 1655 } else { 1656 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, 1657 zero, arena); 1658 } 1659 1660 return (usize); 1661 } 1662 1663 JEMALLOC_ALWAYS_INLINE_C size_t 1664 ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra, 1665 size_t alignment, size_t max_usize, bool zero, arena_t *arena, 1666 prof_thr_cnt_t *cnt) 1667 { 1668 size_t usize; 1669 prof_ctx_t *old_ctx; 1670 1671 old_ctx = prof_ctx_get(ptr); 1672 if ((uintptr_t)cnt != (uintptr_t)1U) { 1673 usize = ixallocx_prof_sample(ptr, old_usize, size, extra, 1674 alignment, zero, max_usize, arena, cnt); 1675 } else { 1676 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, 1677 zero, arena); 1678 } 1679 if (usize == old_usize) 1680 return (usize); 1681 prof_realloc(ptr, usize, cnt, old_usize, old_ctx); 1682 1683 return (usize); 1684 } 1685 1686 size_t 1687 je_xallocx(void *ptr, size_t size, size_t extra, int flags) 1688 { 1689 size_t usize, old_usize; 1690 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1691 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) 1692 & (SIZE_T_MAX-1)); 1693 bool zero = flags & MALLOCX_ZERO; 1694 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; 1695 arena_t *arena; 1696 1697 assert(ptr != NULL); 1698 assert(size != 0); 1699 assert(SIZE_T_MAX - size >= extra); 1700 assert(malloc_initialized || IS_INITIALIZER); 1701 malloc_thread_init(); 1702 1703 if (arena_ind != UINT_MAX) 1704 arena = arenas[arena_ind]; 1705 else 1706 arena = NULL; 1707 1708 old_usize = isalloc(ptr, config_prof); 1709 if (config_valgrind && in_valgrind) 1710 old_rzsize = u2rz(old_usize); 1711 1712 if (config_prof && opt_prof) { 1713 prof_thr_cnt_t *cnt; 1714 /* 1715 * usize isn't knowable before ixalloc() returns when extra is 1716 * non-zero. Therefore, compute its maximum possible value and 1717 * use that in PROF_ALLOC_PREP() to decide whether to capture a 1718 * backtrace. prof_realloc() will use the actual usize to 1719 * decide whether to sample. 1720 */ 1721 size_t max_usize = (alignment == 0) ? s2u(size+extra) : 1722 sa2u(size+extra, alignment); 1723 PROF_ALLOC_PREP(max_usize, cnt); 1724 usize = ixallocx_prof(ptr, old_usize, size, extra, alignment, 1725 max_usize, zero, arena, cnt); 1726 } else { 1727 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, 1728 zero, arena); 1729 } 1730 if (usize == old_usize) 1731 goto label_not_resized; 1732 1733 if (config_stats) { 1734 thread_allocated_t *ta; 1735 ta = thread_allocated_tsd_get(); 1736 ta->allocated += usize; 1737 ta->deallocated += old_usize; 1738 } 1739 JEMALLOC_VALGRIND_REALLOC(false, ptr, usize, false, ptr, old_usize, 1740 old_rzsize, false, zero); 1741 label_not_resized: 1742 UTRACE(ptr, size, ptr); 1743 return (usize); 1744 } 1745 1746 size_t 1747 je_sallocx(const void *ptr, int flags) 1748 { 1749 size_t usize; 1750 1751 assert(malloc_initialized || IS_INITIALIZER); 1752 malloc_thread_init(); 1753 1754 if (config_ivsalloc) 1755 usize = ivsalloc(ptr, config_prof); 1756 else { 1757 assert(ptr != NULL); 1758 usize = isalloc(ptr, config_prof); 1759 } 1760 1761 return (usize); 1762 } 1763 1764 void 1765 je_dallocx(void *ptr, int flags) 1766 { 1767 size_t usize; 1768 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1769 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; 1770 bool try_tcache; 1771 1772 assert(ptr != NULL); 1773 assert(malloc_initialized || IS_INITIALIZER); 1774 1775 if (arena_ind != UINT_MAX) { 1776 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1777 try_tcache = (chunk == ptr || chunk->arena != 1778 arenas[arena_ind]); 1779 } else 1780 try_tcache = true; 1781 1782 UTRACE(ptr, 0, 0); 1783 if (config_stats || config_valgrind) 1784 usize = isalloc(ptr, config_prof); 1785 if (config_prof && opt_prof) { 1786 if (config_stats == false && config_valgrind == false) 1787 usize = isalloc(ptr, config_prof); 1788 prof_free(ptr, usize); 1789 } 1790 if (config_stats) 1791 thread_allocated_tsd_get()->deallocated += usize; 1792 if (config_valgrind && in_valgrind) 1793 rzsize = p2rz(ptr); 1794 iqalloct(ptr, try_tcache); 1795 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1796 } 1797 1798 size_t 1799 je_nallocx(size_t size, int flags) 1800 { 1801 size_t usize; 1802 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) 1803 & (SIZE_T_MAX-1)); 1804 1805 assert(size != 0); 1806 1807 if (malloc_init()) 1808 return (0); 1809 1810 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 1811 assert(usize != 0); 1812 return (usize); 1813 } 1814 1815 int 1816 je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, 1817 size_t newlen) 1818 { 1819 1820 if (malloc_init()) 1821 return (EAGAIN); 1822 1823 return (ctl_byname(name, oldp, oldlenp, newp, newlen)); 1824 } 1825 1826 int 1827 je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) 1828 { 1829 1830 if (malloc_init()) 1831 return (EAGAIN); 1832 1833 return (ctl_nametomib(name, mibp, miblenp)); 1834 } 1835 1836 int 1837 je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 1838 void *newp, size_t newlen) 1839 { 1840 1841 if (malloc_init()) 1842 return (EAGAIN); 1843 1844 return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); 1845 } 1846 1847 void 1848 je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, 1849 const char *opts) 1850 { 1851 1852 stats_print(write_cb, cbopaque, opts); 1853 } 1854 1855 size_t 1856 je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) 1857 { 1858 size_t ret; 1859 1860 assert(malloc_initialized || IS_INITIALIZER); 1861 malloc_thread_init(); 1862 1863 if (config_ivsalloc) 1864 ret = ivsalloc(ptr, config_prof); 1865 else 1866 ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0; 1867 1868 return (ret); 1869 } 1870 1871 /* 1872 * End non-standard functions. 1873 */ 1874 /******************************************************************************/ 1875 /* 1876 * The following functions are used by threading libraries for protection of 1877 * malloc during fork(). 1878 */ 1879 1880 /* 1881 * If an application creates a thread before doing any allocation in the main 1882 * thread, then calls fork(2) in the main thread followed by memory allocation 1883 * in the child process, a race can occur that results in deadlock within the 1884 * child: the main thread may have forked while the created thread had 1885 * partially initialized the allocator. Ordinarily jemalloc prevents 1886 * fork/malloc races via the following functions it registers during 1887 * initialization using pthread_atfork(), but of course that does no good if 1888 * the allocator isn't fully initialized at fork time. The following library 1889 * constructor is a partial solution to this problem. It may still possible to 1890 * trigger the deadlock described above, but doing so would involve forking via 1891 * a library constructor that runs before jemalloc's runs. 1892 */ 1893 JEMALLOC_ATTR(constructor) 1894 static void 1895 jemalloc_constructor(void) 1896 { 1897 1898 malloc_init(); 1899 } 1900 1901 #ifndef JEMALLOC_MUTEX_INIT_CB 1902 void 1903 jemalloc_prefork(void) 1904 #else 1905 JEMALLOC_EXPORT void 1906 _malloc_prefork(void) 1907 #endif 1908 { 1909 unsigned i; 1910 1911 #ifdef JEMALLOC_MUTEX_INIT_CB 1912 if (malloc_initialized == false) 1913 return; 1914 #endif 1915 assert(malloc_initialized); 1916 1917 /* Acquire all mutexes in a safe order. */ 1918 ctl_prefork(); 1919 prof_prefork(); 1920 malloc_mutex_prefork(&arenas_lock); 1921 for (i = 0; i < narenas_total; i++) { 1922 if (arenas[i] != NULL) 1923 arena_prefork(arenas[i]); 1924 } 1925 chunk_prefork(); 1926 base_prefork(); 1927 huge_prefork(); 1928 } 1929 1930 #ifndef JEMALLOC_MUTEX_INIT_CB 1931 void 1932 jemalloc_postfork_parent(void) 1933 #else 1934 JEMALLOC_EXPORT void 1935 _malloc_postfork(void) 1936 #endif 1937 { 1938 unsigned i; 1939 1940 #ifdef JEMALLOC_MUTEX_INIT_CB 1941 if (malloc_initialized == false) 1942 return; 1943 #endif 1944 assert(malloc_initialized); 1945 1946 /* Release all mutexes, now that fork() has completed. */ 1947 huge_postfork_parent(); 1948 base_postfork_parent(); 1949 chunk_postfork_parent(); 1950 for (i = 0; i < narenas_total; i++) { 1951 if (arenas[i] != NULL) 1952 arena_postfork_parent(arenas[i]); 1953 } 1954 malloc_mutex_postfork_parent(&arenas_lock); 1955 prof_postfork_parent(); 1956 ctl_postfork_parent(); 1957 } 1958 1959 void 1960 jemalloc_postfork_child(void) 1961 { 1962 unsigned i; 1963 1964 assert(malloc_initialized); 1965 1966 /* Release all mutexes, now that fork() has completed. */ 1967 huge_postfork_child(); 1968 base_postfork_child(); 1969 chunk_postfork_child(); 1970 for (i = 0; i < narenas_total; i++) { 1971 if (arenas[i] != NULL) 1972 arena_postfork_child(arenas[i]); 1973 } 1974 malloc_mutex_postfork_child(&arenas_lock); 1975 prof_postfork_child(); 1976 ctl_postfork_child(); 1977 } 1978 1979 /******************************************************************************/ 1980 /* 1981 * The following functions are used for TLS allocation/deallocation in static 1982 * binaries on FreeBSD. The primary difference between these and i[mcd]alloc() 1983 * is that these avoid accessing TLS variables. 1984 */ 1985 1986 static void * 1987 a0alloc(size_t size, bool zero) 1988 { 1989 1990 if (malloc_init()) 1991 return (NULL); 1992 1993 if (size == 0) 1994 size = 1; 1995 1996 if (size <= arena_maxclass) 1997 return (arena_malloc(arenas[0], size, zero, false)); 1998 else 1999 return (huge_malloc(NULL, size, zero)); 2000 } 2001 2002 void * 2003 a0malloc(size_t size) 2004 { 2005 2006 return (a0alloc(size, false)); 2007 } 2008 2009 void * 2010 a0calloc(size_t num, size_t size) 2011 { 2012 2013 return (a0alloc(num * size, true)); 2014 } 2015 2016 void 2017 a0free(void *ptr) 2018 { 2019 arena_chunk_t *chunk; 2020 2021 if (ptr == NULL) 2022 return; 2023 2024 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 2025 if (chunk != ptr) 2026 arena_dalloc(chunk, ptr, false); 2027 else 2028 huge_dalloc(ptr); 2029 } 2030 2031 /******************************************************************************/ 2032
