Home | History | Annotate | Download | only in src
      1 #define	JEMALLOC_C_
      2 #include "jemalloc/internal/jemalloc_internal.h"
      3 
      4 /******************************************************************************/
      5 /* Data. */
      6 
      7 /* Runtime configuration options. */
      8 const char	*je_malloc_conf
      9 #ifndef _WIN32
     10     JEMALLOC_ATTR(weak)
     11 #endif
     12     ;
     13 bool	opt_abort =
     14 #ifdef JEMALLOC_DEBUG
     15     true
     16 #else
     17     false
     18 #endif
     19     ;
     20 const char	*opt_junk =
     21 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
     22     "true"
     23 #else
     24     "false"
     25 #endif
     26     ;
     27 bool	opt_junk_alloc =
     28 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
     29     true
     30 #else
     31     false
     32 #endif
     33     ;
     34 bool	opt_junk_free =
     35 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
     36     true
     37 #else
     38     false
     39 #endif
     40     ;
     41 
     42 size_t	opt_quarantine = ZU(0);
     43 bool	opt_redzone = false;
     44 bool	opt_utrace = false;
     45 bool	opt_xmalloc = false;
     46 bool	opt_zero = false;
     47 unsigned	opt_narenas = 0;
     48 
     49 /* Initialized to true if the process is running inside Valgrind. */
     50 bool	in_valgrind;
     51 
     52 unsigned	ncpus;
     53 
     54 /* Protects arenas initialization. */
     55 static malloc_mutex_t	arenas_lock;
     56 /*
     57  * Arenas that are used to service external requests.  Not all elements of the
     58  * arenas array are necessarily used; arenas are created lazily as needed.
     59  *
     60  * arenas[0..narenas_auto) are used for automatic multiplexing of threads and
     61  * arenas.  arenas[narenas_auto..narenas_total) are only used if the application
     62  * takes some action to create them and allocate from them.
     63  */
     64 arena_t			**arenas;
     65 static unsigned		narenas_total; /* Use narenas_total_*(). */
     66 static arena_t		*a0; /* arenas[0]; read-only after initialization. */
     67 unsigned		narenas_auto; /* Read-only after initialization. */
     68 
     69 typedef enum {
     70 	malloc_init_uninitialized	= 3,
     71 	malloc_init_a0_initialized	= 2,
     72 	malloc_init_recursible		= 1,
     73 	malloc_init_initialized		= 0 /* Common case --> jnz. */
     74 } malloc_init_t;
     75 static malloc_init_t	malloc_init_state = malloc_init_uninitialized;
     76 
     77 /* False should be the common case.  Set to true to trigger initialization. */
     78 static bool	malloc_slow = true;
     79 
     80 /* When malloc_slow is true, set the corresponding bits for sanity check. */
     81 enum {
     82 	flag_opt_junk_alloc	= (1U),
     83 	flag_opt_junk_free	= (1U << 1),
     84 	flag_opt_quarantine	= (1U << 2),
     85 	flag_opt_zero		= (1U << 3),
     86 	flag_opt_utrace		= (1U << 4),
     87 	flag_in_valgrind	= (1U << 5),
     88 	flag_opt_xmalloc	= (1U << 6)
     89 };
     90 static uint8_t	malloc_slow_flags;
     91 
     92 JEMALLOC_ALIGNED(CACHELINE)
     93 const size_t	pind2sz_tab[NPSIZES] = {
     94 #define	PSZ_yes(lg_grp, ndelta, lg_delta)				\
     95 	(((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta))),
     96 #define	PSZ_no(lg_grp, ndelta, lg_delta)
     97 #define	SC(index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup)	\
     98 	PSZ_##psz(lg_grp, ndelta, lg_delta)
     99 	SIZE_CLASSES
    100 #undef PSZ_yes
    101 #undef PSZ_no
    102 #undef SC
    103 };
    104 
    105 JEMALLOC_ALIGNED(CACHELINE)
    106 const size_t	index2size_tab[NSIZES] = {
    107 #define	SC(index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup)	\
    108 	((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)),
    109 	SIZE_CLASSES
    110 #undef SC
    111 };
    112 
    113 JEMALLOC_ALIGNED(CACHELINE)
    114 const uint8_t	size2index_tab[] = {
    115 #if LG_TINY_MIN == 0
    116 #warning "Dangerous LG_TINY_MIN"
    117 #define	S2B_0(i)	i,
    118 #elif LG_TINY_MIN == 1
    119 #warning "Dangerous LG_TINY_MIN"
    120 #define	S2B_1(i)	i,
    121 #elif LG_TINY_MIN == 2
    122 #warning "Dangerous LG_TINY_MIN"
    123 #define	S2B_2(i)	i,
    124 #elif LG_TINY_MIN == 3
    125 #define	S2B_3(i)	i,
    126 #elif LG_TINY_MIN == 4
    127 #define	S2B_4(i)	i,
    128 #elif LG_TINY_MIN == 5
    129 #define	S2B_5(i)	i,
    130 #elif LG_TINY_MIN == 6
    131 #define	S2B_6(i)	i,
    132 #elif LG_TINY_MIN == 7
    133 #define	S2B_7(i)	i,
    134 #elif LG_TINY_MIN == 8
    135 #define	S2B_8(i)	i,
    136 #elif LG_TINY_MIN == 9
    137 #define	S2B_9(i)	i,
    138 #elif LG_TINY_MIN == 10
    139 #define	S2B_10(i)	i,
    140 #elif LG_TINY_MIN == 11
    141 #define	S2B_11(i)	i,
    142 #else
    143 #error "Unsupported LG_TINY_MIN"
    144 #endif
    145 #if LG_TINY_MIN < 1
    146 #define	S2B_1(i)	S2B_0(i) S2B_0(i)
    147 #endif
    148 #if LG_TINY_MIN < 2
    149 #define	S2B_2(i)	S2B_1(i) S2B_1(i)
    150 #endif
    151 #if LG_TINY_MIN < 3
    152 #define	S2B_3(i)	S2B_2(i) S2B_2(i)
    153 #endif
    154 #if LG_TINY_MIN < 4
    155 #define	S2B_4(i)	S2B_3(i) S2B_3(i)
    156 #endif
    157 #if LG_TINY_MIN < 5
    158 #define	S2B_5(i)	S2B_4(i) S2B_4(i)
    159 #endif
    160 #if LG_TINY_MIN < 6
    161 #define	S2B_6(i)	S2B_5(i) S2B_5(i)
    162 #endif
    163 #if LG_TINY_MIN < 7
    164 #define	S2B_7(i)	S2B_6(i) S2B_6(i)
    165 #endif
    166 #if LG_TINY_MIN < 8
    167 #define	S2B_8(i)	S2B_7(i) S2B_7(i)
    168 #endif
    169 #if LG_TINY_MIN < 9
    170 #define	S2B_9(i)	S2B_8(i) S2B_8(i)
    171 #endif
    172 #if LG_TINY_MIN < 10
    173 #define	S2B_10(i)	S2B_9(i) S2B_9(i)
    174 #endif
    175 #if LG_TINY_MIN < 11
    176 #define	S2B_11(i)	S2B_10(i) S2B_10(i)
    177 #endif
    178 #define	S2B_no(i)
    179 #define	SC(index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup)	\
    180 	S2B_##lg_delta_lookup(index)
    181 	SIZE_CLASSES
    182 #undef S2B_3
    183 #undef S2B_4
    184 #undef S2B_5
    185 #undef S2B_6
    186 #undef S2B_7
    187 #undef S2B_8
    188 #undef S2B_9
    189 #undef S2B_10
    190 #undef S2B_11
    191 #undef S2B_no
    192 #undef SC
    193 };
    194 
    195 #ifdef JEMALLOC_THREADED_INIT
    196 /* Used to let the initializing thread recursively allocate. */
    197 #  define NO_INITIALIZER	((unsigned long)0)
    198 #  define INITIALIZER		pthread_self()
    199 #  define IS_INITIALIZER	(malloc_initializer == pthread_self())
    200 static pthread_t		malloc_initializer = NO_INITIALIZER;
    201 #else
    202 #  define NO_INITIALIZER	false
    203 #  define INITIALIZER		true
    204 #  define IS_INITIALIZER	malloc_initializer
    205 static bool			malloc_initializer = NO_INITIALIZER;
    206 #endif
    207 
    208 /* Used to avoid initialization races. */
    209 #ifdef _WIN32
    210 #if _WIN32_WINNT >= 0x0600
    211 static malloc_mutex_t	init_lock = SRWLOCK_INIT;
    212 #else
    213 static malloc_mutex_t	init_lock;
    214 static bool init_lock_initialized = false;
    215 
    216 JEMALLOC_ATTR(constructor)
    217 static void WINAPI
    218 _init_init_lock(void)
    219 {
    220 
    221 	/* If another constructor in the same binary is using mallctl to
    222 	 * e.g. setup chunk hooks, it may end up running before this one,
    223 	 * and malloc_init_hard will crash trying to lock the uninitialized
    224 	 * lock. So we force an initialization of the lock in
    225 	 * malloc_init_hard as well. We don't try to care about atomicity
    226 	 * of the accessed to the init_lock_initialized boolean, since it
    227 	 * really only matters early in the process creation, before any
    228 	 * separate thread normally starts doing anything. */
    229 	if (!init_lock_initialized)
    230 		malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT);
    231 	init_lock_initialized = true;
    232 }
    233 
    234 #ifdef _MSC_VER
    235 #  pragma section(".CRT$XCU", read)
    236 JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used)
    237 static const void (WINAPI *init_init_lock)(void) = _init_init_lock;
    238 #endif
    239 #endif
    240 #else
    241 static malloc_mutex_t	init_lock = MALLOC_MUTEX_INITIALIZER;
    242 #endif
    243 
    244 typedef struct {
    245 	void	*p;	/* Input pointer (as in realloc(p, s)). */
    246 	size_t	s;	/* Request size. */
    247 	void	*r;	/* Result pointer. */
    248 } malloc_utrace_t;
    249 
    250 #ifdef JEMALLOC_UTRACE
    251 #  define UTRACE(a, b, c) do {						\
    252 	if (unlikely(opt_utrace)) {					\
    253 		int utrace_serrno = errno;				\
    254 		malloc_utrace_t ut;					\
    255 		ut.p = (a);						\
    256 		ut.s = (b);						\
    257 		ut.r = (c);						\
    258 		utrace(&ut, sizeof(ut));				\
    259 		errno = utrace_serrno;					\
    260 	}								\
    261 } while (0)
    262 #else
    263 #  define UTRACE(a, b, c)
    264 #endif
    265 
    266 /******************************************************************************/
    267 /*
    268  * Function prototypes for static functions that are referenced prior to
    269  * definition.
    270  */
    271 
    272 static bool	malloc_init_hard_a0(void);
    273 static bool	malloc_init_hard(void);
    274 
    275 /******************************************************************************/
    276 /*
    277  * Begin miscellaneous support functions.
    278  */
    279 
    280 JEMALLOC_ALWAYS_INLINE_C bool
    281 malloc_initialized(void)
    282 {
    283 
    284 	return (malloc_init_state == malloc_init_initialized);
    285 }
    286 
    287 JEMALLOC_ALWAYS_INLINE_C void
    288 malloc_thread_init(void)
    289 {
    290 
    291 	/*
    292 	 * TSD initialization can't be safely done as a side effect of
    293 	 * deallocation, because it is possible for a thread to do nothing but
    294 	 * deallocate its TLS data via free(), in which case writing to TLS
    295 	 * would cause write-after-free memory corruption.  The quarantine
    296 	 * facility *only* gets used as a side effect of deallocation, so make
    297 	 * a best effort attempt at initializing its TSD by hooking all
    298 	 * allocation events.
    299 	 */
    300 	if (config_fill && unlikely(opt_quarantine))
    301 		quarantine_alloc_hook();
    302 }
    303 
    304 JEMALLOC_ALWAYS_INLINE_C bool
    305 malloc_init_a0(void)
    306 {
    307 
    308 	if (unlikely(malloc_init_state == malloc_init_uninitialized))
    309 		return (malloc_init_hard_a0());
    310 	return (false);
    311 }
    312 
    313 JEMALLOC_ALWAYS_INLINE_C bool
    314 malloc_init(void)
    315 {
    316 
    317 	if (unlikely(!malloc_initialized()) && malloc_init_hard())
    318 		return (true);
    319 	malloc_thread_init();
    320 
    321 	return (false);
    322 }
    323 
    324 /*
    325  * The a0*() functions are used instead of i{d,}alloc() in situations that
    326  * cannot tolerate TLS variable access.
    327  */
    328 
    329 static void *
    330 a0ialloc(size_t size, bool zero, bool is_metadata)
    331 {
    332 
    333 	if (unlikely(malloc_init_a0()))
    334 		return (NULL);
    335 
    336 	return (iallocztm(TSDN_NULL, size, size2index(size), zero, NULL,
    337 	    is_metadata, arena_get(TSDN_NULL, 0, true), true));
    338 }
    339 
    340 static void
    341 a0idalloc(void *ptr, bool is_metadata)
    342 {
    343 
    344 	idalloctm(TSDN_NULL, ptr, false, is_metadata, true);
    345 }
    346 
    347 arena_t *
    348 a0get(void)
    349 {
    350 
    351 	return (a0);
    352 }
    353 
    354 void *
    355 a0malloc(size_t size)
    356 {
    357 
    358 	return (a0ialloc(size, false, true));
    359 }
    360 
    361 void
    362 a0dalloc(void *ptr)
    363 {
    364 
    365 	a0idalloc(ptr, true);
    366 }
    367 
    368 /*
    369  * FreeBSD's libc uses the bootstrap_*() functions in bootstrap-senstive
    370  * situations that cannot tolerate TLS variable access (TLS allocation and very
    371  * early internal data structure initialization).
    372  */
    373 
    374 void *
    375 bootstrap_malloc(size_t size)
    376 {
    377 
    378 	if (unlikely(size == 0))
    379 		size = 1;
    380 
    381 	return (a0ialloc(size, false, false));
    382 }
    383 
    384 void *
    385 bootstrap_calloc(size_t num, size_t size)
    386 {
    387 	size_t num_size;
    388 
    389 	num_size = num * size;
    390 	if (unlikely(num_size == 0)) {
    391 		assert(num == 0 || size == 0);
    392 		num_size = 1;
    393 	}
    394 
    395 	return (a0ialloc(num_size, true, false));
    396 }
    397 
    398 void
    399 bootstrap_free(void *ptr)
    400 {
    401 
    402 	if (unlikely(ptr == NULL))
    403 		return;
    404 
    405 	a0idalloc(ptr, false);
    406 }
    407 
    408 static void
    409 arena_set(unsigned ind, arena_t *arena)
    410 {
    411 
    412 	atomic_write_p((void **)&arenas[ind], arena);
    413 }
    414 
    415 static void
    416 narenas_total_set(unsigned narenas)
    417 {
    418 
    419 	atomic_write_u(&narenas_total, narenas);
    420 }
    421 
    422 static void
    423 narenas_total_inc(void)
    424 {
    425 
    426 	atomic_add_u(&narenas_total, 1);
    427 }
    428 
    429 unsigned
    430 narenas_total_get(void)
    431 {
    432 
    433 	return (atomic_read_u(&narenas_total));
    434 }
    435 
    436 /* Create a new arena and insert it into the arenas array at index ind. */
    437 static arena_t *
    438 arena_init_locked(tsdn_t *tsdn, unsigned ind)
    439 {
    440 	arena_t *arena;
    441 
    442 	assert(ind <= narenas_total_get());
    443 	if (ind > MALLOCX_ARENA_MAX)
    444 		return (NULL);
    445 	if (ind == narenas_total_get())
    446 		narenas_total_inc();
    447 
    448 	/*
    449 	 * Another thread may have already initialized arenas[ind] if it's an
    450 	 * auto arena.
    451 	 */
    452 	arena = arena_get(tsdn, ind, false);
    453 	if (arena != NULL) {
    454 		assert(ind < narenas_auto);
    455 		return (arena);
    456 	}
    457 
    458 	/* Actually initialize the arena. */
    459 	arena = arena_new(tsdn, ind);
    460 	arena_set(ind, arena);
    461 	return (arena);
    462 }
    463 
    464 arena_t *
    465 arena_init(tsdn_t *tsdn, unsigned ind)
    466 {
    467 	arena_t *arena;
    468 
    469 	malloc_mutex_lock(tsdn, &arenas_lock);
    470 	arena = arena_init_locked(tsdn, ind);
    471 	malloc_mutex_unlock(tsdn, &arenas_lock);
    472 	return (arena);
    473 }
    474 
    475 static void
    476 arena_bind(tsd_t *tsd, unsigned ind, bool internal)
    477 {
    478 	arena_t *arena;
    479 
    480 	if (!tsd_nominal(tsd))
    481 		return;
    482 
    483 	arena = arena_get(tsd_tsdn(tsd), ind, false);
    484 	arena_nthreads_inc(arena, internal);
    485 
    486 	if (internal)
    487 		tsd_iarena_set(tsd, arena);
    488 	else
    489 		tsd_arena_set(tsd, arena);
    490 }
    491 
    492 void
    493 arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind)
    494 {
    495 	arena_t *oldarena, *newarena;
    496 
    497 	oldarena = arena_get(tsd_tsdn(tsd), oldind, false);
    498 	newarena = arena_get(tsd_tsdn(tsd), newind, false);
    499 	arena_nthreads_dec(oldarena, false);
    500 	arena_nthreads_inc(newarena, false);
    501 	tsd_arena_set(tsd, newarena);
    502 }
    503 
    504 static void
    505 arena_unbind(tsd_t *tsd, unsigned ind, bool internal)
    506 {
    507 	arena_t *arena;
    508 
    509 	arena = arena_get(tsd_tsdn(tsd), ind, false);
    510 	arena_nthreads_dec(arena, internal);
    511 	if (internal)
    512 		tsd_iarena_set(tsd, NULL);
    513 	else
    514 		tsd_arena_set(tsd, NULL);
    515 }
    516 
    517 arena_tdata_t *
    518 arena_tdata_get_hard(tsd_t *tsd, unsigned ind)
    519 {
    520 	arena_tdata_t *tdata, *arenas_tdata_old;
    521 	arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd);
    522 	unsigned narenas_tdata_old, i;
    523 	unsigned narenas_tdata = tsd_narenas_tdata_get(tsd);
    524 	unsigned narenas_actual = narenas_total_get();
    525 
    526 	/*
    527 	 * Dissociate old tdata array (and set up for deallocation upon return)
    528 	 * if it's too small.
    529 	 */
    530 	if (arenas_tdata != NULL && narenas_tdata < narenas_actual) {
    531 		arenas_tdata_old = arenas_tdata;
    532 		narenas_tdata_old = narenas_tdata;
    533 		arenas_tdata = NULL;
    534 		narenas_tdata = 0;
    535 		tsd_arenas_tdata_set(tsd, arenas_tdata);
    536 		tsd_narenas_tdata_set(tsd, narenas_tdata);
    537 	} else {
    538 		arenas_tdata_old = NULL;
    539 		narenas_tdata_old = 0;
    540 	}
    541 
    542 	/* Allocate tdata array if it's missing. */
    543 	if (arenas_tdata == NULL) {
    544 		bool *arenas_tdata_bypassp = tsd_arenas_tdata_bypassp_get(tsd);
    545 		narenas_tdata = (ind < narenas_actual) ? narenas_actual : ind+1;
    546 
    547 		if (tsd_nominal(tsd) && !*arenas_tdata_bypassp) {
    548 			*arenas_tdata_bypassp = true;
    549 			arenas_tdata = (arena_tdata_t *)a0malloc(
    550 			    sizeof(arena_tdata_t) * narenas_tdata);
    551 			*arenas_tdata_bypassp = false;
    552 		}
    553 		if (arenas_tdata == NULL) {
    554 			tdata = NULL;
    555 			goto label_return;
    556 		}
    557 		assert(tsd_nominal(tsd) && !*arenas_tdata_bypassp);
    558 		tsd_arenas_tdata_set(tsd, arenas_tdata);
    559 		tsd_narenas_tdata_set(tsd, narenas_tdata);
    560 	}
    561 
    562 	/*
    563 	 * Copy to tdata array.  It's possible that the actual number of arenas
    564 	 * has increased since narenas_total_get() was called above, but that
    565 	 * causes no correctness issues unless two threads concurrently execute
    566 	 * the arenas.extend mallctl, which we trust mallctl synchronization to
    567 	 * prevent.
    568 	 */
    569 
    570 	/* Copy/initialize tickers. */
    571 	for (i = 0; i < narenas_actual; i++) {
    572 		if (i < narenas_tdata_old) {
    573 			ticker_copy(&arenas_tdata[i].decay_ticker,
    574 			    &arenas_tdata_old[i].decay_ticker);
    575 		} else {
    576 			ticker_init(&arenas_tdata[i].decay_ticker,
    577 			    DECAY_NTICKS_PER_UPDATE);
    578 		}
    579 	}
    580 	if (narenas_tdata > narenas_actual) {
    581 		memset(&arenas_tdata[narenas_actual], 0, sizeof(arena_tdata_t)
    582 		    * (narenas_tdata - narenas_actual));
    583 	}
    584 
    585 	/* Read the refreshed tdata array. */
    586 	tdata = &arenas_tdata[ind];
    587 label_return:
    588 	if (arenas_tdata_old != NULL)
    589 		a0dalloc(arenas_tdata_old);
    590 	return (tdata);
    591 }
    592 
    593 /* Slow path, called only by arena_choose(). */
    594 arena_t *
    595 arena_choose_hard(tsd_t *tsd, bool internal)
    596 {
    597 	arena_t *ret JEMALLOC_CC_SILENCE_INIT(NULL);
    598 
    599 	if (narenas_auto > 1) {
    600 		unsigned i, j, choose[2], first_null;
    601 
    602 		/*
    603 		 * Determine binding for both non-internal and internal
    604 		 * allocation.
    605 		 *
    606 		 *   choose[0]: For application allocation.
    607 		 *   choose[1]: For internal metadata allocation.
    608 		 */
    609 
    610 		for (j = 0; j < 2; j++)
    611 			choose[j] = 0;
    612 
    613 		first_null = narenas_auto;
    614 		malloc_mutex_lock(tsd_tsdn(tsd), &arenas_lock);
    615 		assert(arena_get(tsd_tsdn(tsd), 0, false) != NULL);
    616 		for (i = 1; i < narenas_auto; i++) {
    617 			if (arena_get(tsd_tsdn(tsd), i, false) != NULL) {
    618 				/*
    619 				 * Choose the first arena that has the lowest
    620 				 * number of threads assigned to it.
    621 				 */
    622 				for (j = 0; j < 2; j++) {
    623 					if (arena_nthreads_get(arena_get(
    624 					    tsd_tsdn(tsd), i, false), !!j) <
    625 					    arena_nthreads_get(arena_get(
    626 					    tsd_tsdn(tsd), choose[j], false),
    627 					    !!j))
    628 						choose[j] = i;
    629 				}
    630 			} else if (first_null == narenas_auto) {
    631 				/*
    632 				 * Record the index of the first uninitialized
    633 				 * arena, in case all extant arenas are in use.
    634 				 *
    635 				 * NB: It is possible for there to be
    636 				 * discontinuities in terms of initialized
    637 				 * versus uninitialized arenas, due to the
    638 				 * "thread.arena" mallctl.
    639 				 */
    640 				first_null = i;
    641 			}
    642 		}
    643 
    644 		for (j = 0; j < 2; j++) {
    645 			if (arena_nthreads_get(arena_get(tsd_tsdn(tsd),
    646 			    choose[j], false), !!j) == 0 || first_null ==
    647 			    narenas_auto) {
    648 				/*
    649 				 * Use an unloaded arena, or the least loaded
    650 				 * arena if all arenas are already initialized.
    651 				 */
    652 				if (!!j == internal) {
    653 					ret = arena_get(tsd_tsdn(tsd),
    654 					    choose[j], false);
    655 				}
    656 			} else {
    657 				arena_t *arena;
    658 
    659 				/* Initialize a new arena. */
    660 				choose[j] = first_null;
    661 				arena = arena_init_locked(tsd_tsdn(tsd),
    662 				    choose[j]);
    663 				if (arena == NULL) {
    664 					malloc_mutex_unlock(tsd_tsdn(tsd),
    665 					    &arenas_lock);
    666 					return (NULL);
    667 				}
    668 				if (!!j == internal)
    669 					ret = arena;
    670 			}
    671 			arena_bind(tsd, choose[j], !!j);
    672 		}
    673 		malloc_mutex_unlock(tsd_tsdn(tsd), &arenas_lock);
    674 	} else {
    675 		ret = arena_get(tsd_tsdn(tsd), 0, false);
    676 		arena_bind(tsd, 0, false);
    677 		arena_bind(tsd, 0, true);
    678 	}
    679 
    680 	return (ret);
    681 }
    682 
    683 void
    684 thread_allocated_cleanup(tsd_t *tsd)
    685 {
    686 
    687 	/* Do nothing. */
    688 }
    689 
    690 void
    691 thread_deallocated_cleanup(tsd_t *tsd)
    692 {
    693 
    694 	/* Do nothing. */
    695 }
    696 
    697 void
    698 iarena_cleanup(tsd_t *tsd)
    699 {
    700 	arena_t *iarena;
    701 
    702 	iarena = tsd_iarena_get(tsd);
    703 	if (iarena != NULL)
    704 		arena_unbind(tsd, iarena->ind, true);
    705 }
    706 
    707 void
    708 arena_cleanup(tsd_t *tsd)
    709 {
    710 	arena_t *arena;
    711 
    712 	arena = tsd_arena_get(tsd);
    713 	if (arena != NULL)
    714 		arena_unbind(tsd, arena->ind, false);
    715 }
    716 
    717 void
    718 arenas_tdata_cleanup(tsd_t *tsd)
    719 {
    720 	arena_tdata_t *arenas_tdata;
    721 
    722 	/* Prevent tsd->arenas_tdata from being (re)created. */
    723 	*tsd_arenas_tdata_bypassp_get(tsd) = true;
    724 
    725 	arenas_tdata = tsd_arenas_tdata_get(tsd);
    726 	if (arenas_tdata != NULL) {
    727 		tsd_arenas_tdata_set(tsd, NULL);
    728 		a0dalloc(arenas_tdata);
    729 	}
    730 }
    731 
    732 void
    733 narenas_tdata_cleanup(tsd_t *tsd)
    734 {
    735 
    736 	/* Do nothing. */
    737 }
    738 
    739 void
    740 arenas_tdata_bypass_cleanup(tsd_t *tsd)
    741 {
    742 
    743 	/* Do nothing. */
    744 }
    745 
    746 static void
    747 stats_print_atexit(void)
    748 {
    749 
    750 	if (config_tcache && config_stats) {
    751 		tsdn_t *tsdn;
    752 		unsigned narenas, i;
    753 
    754 		tsdn = tsdn_fetch();
    755 
    756 		/*
    757 		 * Merge stats from extant threads.  This is racy, since
    758 		 * individual threads do not lock when recording tcache stats
    759 		 * events.  As a consequence, the final stats may be slightly
    760 		 * out of date by the time they are reported, if other threads
    761 		 * continue to allocate.
    762 		 */
    763 		for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
    764 			arena_t *arena = arena_get(tsdn, i, false);
    765 			if (arena != NULL) {
    766 				tcache_t *tcache;
    767 
    768 				/*
    769 				 * tcache_stats_merge() locks bins, so if any
    770 				 * code is introduced that acquires both arena
    771 				 * and bin locks in the opposite order,
    772 				 * deadlocks may result.
    773 				 */
    774 				malloc_mutex_lock(tsdn, &arena->lock);
    775 				ql_foreach(tcache, &arena->tcache_ql, link) {
    776 					tcache_stats_merge(tsdn, tcache, arena);
    777 				}
    778 				malloc_mutex_unlock(tsdn, &arena->lock);
    779 			}
    780 		}
    781 	}
    782 	je_malloc_stats_print(NULL, NULL, NULL);
    783 }
    784 
    785 /*
    786  * End miscellaneous support functions.
    787  */
    788 /******************************************************************************/
    789 /*
    790  * Begin initialization functions.
    791  */
    792 
    793 #ifndef JEMALLOC_HAVE_SECURE_GETENV
    794 static char *
    795 secure_getenv(const char *name)
    796 {
    797 
    798 #  ifdef JEMALLOC_HAVE_ISSETUGID
    799 	if (issetugid() != 0)
    800 		return (NULL);
    801 #  endif
    802 	return (getenv(name));
    803 }
    804 #endif
    805 
    806 static unsigned
    807 malloc_ncpus(void)
    808 {
    809 	long result;
    810 
    811 #ifdef _WIN32
    812 	SYSTEM_INFO si;
    813 	GetSystemInfo(&si);
    814 	result = si.dwNumberOfProcessors;
    815 #elif defined(JEMALLOC_GLIBC_MALLOC_HOOK) && defined(CPU_COUNT)
    816 	/*
    817 	 * glibc >= 2.6 has the CPU_COUNT macro.
    818 	 *
    819 	 * glibc's sysconf() uses isspace().  glibc allocates for the first time
    820 	 * *before* setting up the isspace tables.  Therefore we need a
    821 	 * different method to get the number of CPUs.
    822 	 */
    823 	{
    824 		cpu_set_t set;
    825 
    826 		pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
    827 		result = CPU_COUNT(&set);
    828 	}
    829 #else
    830 	result = sysconf(_SC_NPROCESSORS_ONLN);
    831 #endif
    832 	return ((result == -1) ? 1 : (unsigned)result);
    833 }
    834 
    835 static bool
    836 malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
    837     char const **v_p, size_t *vlen_p)
    838 {
    839 	bool accept;
    840 	const char *opts = *opts_p;
    841 
    842 	*k_p = opts;
    843 
    844 	for (accept = false; !accept;) {
    845 		switch (*opts) {
    846 		case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
    847 		case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
    848 		case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
    849 		case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
    850 		case 'Y': case 'Z':
    851 		case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
    852 		case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
    853 		case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
    854 		case 's': case 't': case 'u': case 'v': case 'w': case 'x':
    855 		case 'y': case 'z':
    856 		case '0': case '1': case '2': case '3': case '4': case '5':
    857 		case '6': case '7': case '8': case '9':
    858 		case '_':
    859 			opts++;
    860 			break;
    861 		case ':':
    862 			opts++;
    863 			*klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p;
    864 			*v_p = opts;
    865 			accept = true;
    866 			break;
    867 		case '\0':
    868 			if (opts != *opts_p) {
    869 				malloc_write("<jemalloc>: Conf string ends "
    870 				    "with key\n");
    871 			}
    872 			return (true);
    873 		default:
    874 			malloc_write("<jemalloc>: Malformed conf string\n");
    875 			return (true);
    876 		}
    877 	}
    878 
    879 	for (accept = false; !accept;) {
    880 		switch (*opts) {
    881 		case ',':
    882 			opts++;
    883 			/*
    884 			 * Look ahead one character here, because the next time
    885 			 * this function is called, it will assume that end of
    886 			 * input has been cleanly reached if no input remains,
    887 			 * but we have optimistically already consumed the
    888 			 * comma if one exists.
    889 			 */
    890 			if (*opts == '\0') {
    891 				malloc_write("<jemalloc>: Conf string ends "
    892 				    "with comma\n");
    893 			}
    894 			*vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p;
    895 			accept = true;
    896 			break;
    897 		case '\0':
    898 			*vlen_p = (uintptr_t)opts - (uintptr_t)*v_p;
    899 			accept = true;
    900 			break;
    901 		default:
    902 			opts++;
    903 			break;
    904 		}
    905 	}
    906 
    907 	*opts_p = opts;
    908 	return (false);
    909 }
    910 
    911 static void
    912 malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v,
    913     size_t vlen)
    914 {
    915 
    916 	malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k,
    917 	    (int)vlen, v);
    918 }
    919 
    920 static void
    921 malloc_slow_flag_init(void)
    922 {
    923 	/*
    924 	 * Combine the runtime options into malloc_slow for fast path.  Called
    925 	 * after processing all the options.
    926 	 */
    927 	malloc_slow_flags |= (opt_junk_alloc ? flag_opt_junk_alloc : 0)
    928 	    | (opt_junk_free ? flag_opt_junk_free : 0)
    929 	    | (opt_quarantine ? flag_opt_quarantine : 0)
    930 	    | (opt_zero ? flag_opt_zero : 0)
    931 	    | (opt_utrace ? flag_opt_utrace : 0)
    932 	    | (opt_xmalloc ? flag_opt_xmalloc : 0);
    933 
    934 	if (config_valgrind)
    935 		malloc_slow_flags |= (in_valgrind ? flag_in_valgrind : 0);
    936 
    937 	malloc_slow = (malloc_slow_flags != 0);
    938 }
    939 
    940 static void
    941 malloc_conf_init(void)
    942 {
    943 	unsigned i;
    944 	char buf[PATH_MAX + 1];
    945 	const char *opts, *k, *v;
    946 	size_t klen, vlen;
    947 
    948 	/*
    949 	 * Automatically configure valgrind before processing options.  The
    950 	 * valgrind option remains in jemalloc 3.x for compatibility reasons.
    951 	 */
    952 	if (config_valgrind) {
    953 		in_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false;
    954 		if (config_fill && unlikely(in_valgrind)) {
    955 			opt_junk = "false";
    956 			opt_junk_alloc = false;
    957 			opt_junk_free = false;
    958 			assert(!opt_zero);
    959 			opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT;
    960 			opt_redzone = true;
    961 		}
    962 		if (config_tcache && unlikely(in_valgrind))
    963 			opt_tcache = false;
    964 	}
    965 
    966 #if defined(__ANDROID__)
    967 	for (i = 0; i < 2; i++) {
    968 #else
    969 	for (i = 0; i < 4; i++) {
    970 #endif
    971 		/* Get runtime configuration. */
    972 		switch (i) {
    973 		case 0:
    974 			opts = config_malloc_conf;
    975 			break;
    976 		case 1:
    977 			if (je_malloc_conf != NULL) {
    978 				/*
    979 				 * Use options that were compiled into the
    980 				 * program.
    981 				 */
    982 				opts = je_malloc_conf;
    983 			} else {
    984 				/* No configuration specified. */
    985 				buf[0] = '\0';
    986 				opts = buf;
    987 			}
    988 			break;
    989 		case 2: {
    990 			ssize_t linklen = 0;
    991 #ifndef _WIN32
    992 			int saved_errno = errno;
    993 			const char *linkname =
    994 #  ifdef JEMALLOC_PREFIX
    995 			    "/etc/"JEMALLOC_PREFIX"malloc.conf"
    996 #  else
    997 			    "/etc/malloc.conf"
    998 #  endif
    999 			    ;
   1000 
   1001 			/*
   1002 			 * Try to use the contents of the "/etc/malloc.conf"
   1003 			 * symbolic link's name.
   1004 			 */
   1005 			linklen = readlink(linkname, buf, sizeof(buf) - 1);
   1006 			if (linklen == -1) {
   1007 				/* No configuration specified. */
   1008 				linklen = 0;
   1009 				/* Restore errno. */
   1010 				set_errno(saved_errno);
   1011 			}
   1012 #endif
   1013 			buf[linklen] = '\0';
   1014 			opts = buf;
   1015 			break;
   1016 		} case 3: {
   1017 			const char *envname =
   1018 #ifdef JEMALLOC_PREFIX
   1019 			    JEMALLOC_CPREFIX"MALLOC_CONF"
   1020 #else
   1021 			    "MALLOC_CONF"
   1022 #endif
   1023 			    ;
   1024 
   1025 			if ((opts = secure_getenv(envname)) != NULL) {
   1026 				/*
   1027 				 * Do nothing; opts is already initialized to
   1028 				 * the value of the MALLOC_CONF environment
   1029 				 * variable.
   1030 				 */
   1031 			} else {
   1032 				/* No configuration specified. */
   1033 				buf[0] = '\0';
   1034 				opts = buf;
   1035 			}
   1036 			break;
   1037 		} default:
   1038 			not_reached();
   1039 			buf[0] = '\0';
   1040 			opts = buf;
   1041 		}
   1042 
   1043 		while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v,
   1044 		    &vlen)) {
   1045 #define	CONF_MATCH(n)							\
   1046 	(sizeof(n)-1 == klen && strncmp(n, k, klen) == 0)
   1047 #define	CONF_MATCH_VALUE(n)						\
   1048 	(sizeof(n)-1 == vlen && strncmp(n, v, vlen) == 0)
   1049 #define	CONF_HANDLE_BOOL(o, n, cont)					\
   1050 			if (CONF_MATCH(n)) {				\
   1051 				if (CONF_MATCH_VALUE("true"))		\
   1052 					o = true;			\
   1053 				else if (CONF_MATCH_VALUE("false"))	\
   1054 					o = false;			\
   1055 				else {					\
   1056 					malloc_conf_error(		\
   1057 					    "Invalid conf value",	\
   1058 					    k, klen, v, vlen);		\
   1059 				}					\
   1060 				if (cont)				\
   1061 					continue;			\
   1062 			}
   1063 #define	CONF_MIN_no(um, min)	false
   1064 #define	CONF_MIN_yes(um, min)	((um) < (min))
   1065 #define	CONF_MAX_no(um, max)	false
   1066 #define	CONF_MAX_yes(um, max)	((um) > (max))
   1067 #define	CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip)	\
   1068 			if (CONF_MATCH(n)) {				\
   1069 				uintmax_t um;				\
   1070 				char *end;				\
   1071 									\
   1072 				set_errno(0);				\
   1073 				um = malloc_strtoumax(v, &end, 0);	\
   1074 				if (get_errno() != 0 || (uintptr_t)end -\
   1075 				    (uintptr_t)v != vlen) {		\
   1076 					malloc_conf_error(		\
   1077 					    "Invalid conf value",	\
   1078 					    k, klen, v, vlen);		\
   1079 				} else if (clip) {			\
   1080 					if (CONF_MIN_##check_min(um,	\
   1081 					    (min)))			\
   1082 						o = (t)(min);		\
   1083 					else if (CONF_MAX_##check_max(	\
   1084 					    um, (max)))			\
   1085 						o = (t)(max);		\
   1086 					else				\
   1087 						o = (t)um;		\
   1088 				} else {				\
   1089 					if (CONF_MIN_##check_min(um,	\
   1090 					    (min)) ||			\
   1091 					    CONF_MAX_##check_max(um,	\
   1092 					    (max))) {			\
   1093 						malloc_conf_error(	\
   1094 						    "Out-of-range "	\
   1095 						    "conf value",	\
   1096 						    k, klen, v, vlen);	\
   1097 					} else				\
   1098 						o = (t)um;		\
   1099 				}					\
   1100 				continue;				\
   1101 			}
   1102 #define	CONF_HANDLE_UNSIGNED(o, n, min, max, check_min, check_max,	\
   1103     clip)								\
   1104 			CONF_HANDLE_T_U(unsigned, o, n, min, max,	\
   1105 			    check_min, check_max, clip)
   1106 #define	CONF_HANDLE_SIZE_T(o, n, min, max, check_min, check_max, clip)	\
   1107 			CONF_HANDLE_T_U(size_t, o, n, min, max,		\
   1108 			    check_min, check_max, clip)
   1109 #define	CONF_HANDLE_SSIZE_T(o, n, min, max)				\
   1110 			if (CONF_MATCH(n)) {				\
   1111 				long l;					\
   1112 				char *end;				\
   1113 									\
   1114 				set_errno(0);				\
   1115 				l = strtol(v, &end, 0);			\
   1116 				if (get_errno() != 0 || (uintptr_t)end -\
   1117 				    (uintptr_t)v != vlen) {		\
   1118 					malloc_conf_error(		\
   1119 					    "Invalid conf value",	\
   1120 					    k, klen, v, vlen);		\
   1121 				} else if (l < (ssize_t)(min) || l >	\
   1122 				    (ssize_t)(max)) {			\
   1123 					malloc_conf_error(		\
   1124 					    "Out-of-range conf value",	\
   1125 					    k, klen, v, vlen);		\
   1126 				} else					\
   1127 					o = l;				\
   1128 				continue;				\
   1129 			}
   1130 #define	CONF_HANDLE_CHAR_P(o, n, d)					\
   1131 			if (CONF_MATCH(n)) {				\
   1132 				size_t cpylen = (vlen <=		\
   1133 				    sizeof(o)-1) ? vlen :		\
   1134 				    sizeof(o)-1;			\
   1135 				strncpy(o, v, cpylen);			\
   1136 				o[cpylen] = '\0';			\
   1137 				continue;				\
   1138 			}
   1139 
   1140 			CONF_HANDLE_BOOL(opt_abort, "abort", true)
   1141 			/*
   1142 			 * Chunks always require at least one header page,
   1143 			 * as many as 2^(LG_SIZE_CLASS_GROUP+1) data pages, and
   1144 			 * possibly an additional page in the presence of
   1145 			 * redzones.  In order to simplify options processing,
   1146 			 * use a conservative bound that accommodates all these
   1147 			 * constraints.
   1148 			 */
   1149 			CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE +
   1150 			    LG_SIZE_CLASS_GROUP + (config_fill ? 2 : 1),
   1151 			    (sizeof(size_t) << 3) - 1, yes, yes, true)
   1152 			if (strncmp("dss", k, klen) == 0) {
   1153 				int i;
   1154 				bool match = false;
   1155 				for (i = 0; i < dss_prec_limit; i++) {
   1156 					if (strncmp(dss_prec_names[i], v, vlen)
   1157 					    == 0) {
   1158 						if (chunk_dss_prec_set(i)) {
   1159 							malloc_conf_error(
   1160 							    "Error setting dss",
   1161 							    k, klen, v, vlen);
   1162 						} else {
   1163 							opt_dss =
   1164 							    dss_prec_names[i];
   1165 							match = true;
   1166 							break;
   1167 						}
   1168 					}
   1169 				}
   1170 				if (!match) {
   1171 					malloc_conf_error("Invalid conf value",
   1172 					    k, klen, v, vlen);
   1173 				}
   1174 				continue;
   1175 			}
   1176 			CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1,
   1177 			    UINT_MAX, yes, no, false)
   1178 			if (strncmp("purge", k, klen) == 0) {
   1179 				int i;
   1180 				bool match = false;
   1181 				for (i = 0; i < purge_mode_limit; i++) {
   1182 					if (strncmp(purge_mode_names[i], v,
   1183 					    vlen) == 0) {
   1184 						opt_purge = (purge_mode_t)i;
   1185 						match = true;
   1186 						break;
   1187 					}
   1188 				}
   1189 				if (!match) {
   1190 					malloc_conf_error("Invalid conf value",
   1191 					    k, klen, v, vlen);
   1192 				}
   1193 				continue;
   1194 			}
   1195 			CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult",
   1196 			    -1, (sizeof(size_t) << 3) - 1)
   1197 			CONF_HANDLE_SSIZE_T(opt_decay_time, "decay_time", -1,
   1198 			    NSTIME_SEC_MAX);
   1199 			CONF_HANDLE_BOOL(opt_stats_print, "stats_print", true)
   1200 			if (config_fill) {
   1201 				if (CONF_MATCH("junk")) {
   1202 					if (CONF_MATCH_VALUE("true")) {
   1203 						if (config_valgrind &&
   1204 						    unlikely(in_valgrind)) {
   1205 							malloc_conf_error(
   1206 							"Deallocation-time "
   1207 							"junk filling cannot "
   1208 							"be enabled while "
   1209 							"running inside "
   1210 							"Valgrind", k, klen, v,
   1211 							vlen);
   1212 						} else {
   1213 							opt_junk = "true";
   1214 							opt_junk_alloc = true;
   1215 							opt_junk_free = true;
   1216 						}
   1217 					} else if (CONF_MATCH_VALUE("false")) {
   1218 						opt_junk = "false";
   1219 						opt_junk_alloc = opt_junk_free =
   1220 						    false;
   1221 					} else if (CONF_MATCH_VALUE("alloc")) {
   1222 						opt_junk = "alloc";
   1223 						opt_junk_alloc = true;
   1224 						opt_junk_free = false;
   1225 					} else if (CONF_MATCH_VALUE("free")) {
   1226 						if (config_valgrind &&
   1227 						    unlikely(in_valgrind)) {
   1228 							malloc_conf_error(
   1229 							"Deallocation-time "
   1230 							"junk filling cannot "
   1231 							"be enabled while "
   1232 							"running inside "
   1233 							"Valgrind", k, klen, v,
   1234 							vlen);
   1235 						} else {
   1236 							opt_junk = "free";
   1237 							opt_junk_alloc = false;
   1238 							opt_junk_free = true;
   1239 						}
   1240 					} else {
   1241 						malloc_conf_error(
   1242 						    "Invalid conf value", k,
   1243 						    klen, v, vlen);
   1244 					}
   1245 					continue;
   1246 				}
   1247 				CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine",
   1248 				    0, SIZE_T_MAX, no, no, false)
   1249 				CONF_HANDLE_BOOL(opt_redzone, "redzone", true)
   1250 				CONF_HANDLE_BOOL(opt_zero, "zero", true)
   1251 			}
   1252 			if (config_utrace) {
   1253 				CONF_HANDLE_BOOL(opt_utrace, "utrace", true)
   1254 			}
   1255 			if (config_xmalloc) {
   1256 				CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc", true)
   1257 			}
   1258 			if (config_tcache) {
   1259 				CONF_HANDLE_BOOL(opt_tcache, "tcache",
   1260 				    !config_valgrind || !in_valgrind)
   1261 				if (CONF_MATCH("tcache")) {
   1262 					assert(config_valgrind && in_valgrind);
   1263 					if (opt_tcache) {
   1264 						opt_tcache = false;
   1265 						malloc_conf_error(
   1266 						"tcache cannot be enabled "
   1267 						"while running inside Valgrind",
   1268 						k, klen, v, vlen);
   1269 					}
   1270 					continue;
   1271 				}
   1272 				CONF_HANDLE_SSIZE_T(opt_lg_tcache_max,
   1273 				    "lg_tcache_max", -1,
   1274 				    (sizeof(size_t) << 3) - 1)
   1275 			}
   1276 			if (config_prof) {
   1277 				CONF_HANDLE_BOOL(opt_prof, "prof", true)
   1278 				CONF_HANDLE_CHAR_P(opt_prof_prefix,
   1279 				    "prof_prefix", "jeprof")
   1280 				CONF_HANDLE_BOOL(opt_prof_active, "prof_active",
   1281 				    true)
   1282 				CONF_HANDLE_BOOL(opt_prof_thread_active_init,
   1283 				    "prof_thread_active_init", true)
   1284 				CONF_HANDLE_SIZE_T(opt_lg_prof_sample,
   1285 				    "lg_prof_sample", 0, (sizeof(uint64_t) << 3)
   1286 				    - 1, no, yes, true)
   1287 				CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum",
   1288 				    true)
   1289 				CONF_HANDLE_SSIZE_T(opt_lg_prof_interval,
   1290 				    "lg_prof_interval", -1,
   1291 				    (sizeof(uint64_t) << 3) - 1)
   1292 				CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump",
   1293 				    true)
   1294 				CONF_HANDLE_BOOL(opt_prof_final, "prof_final",
   1295 				    true)
   1296 				CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak",
   1297 				    true)
   1298 			}
   1299 			malloc_conf_error("Invalid conf pair", k, klen, v,
   1300 			    vlen);
   1301 #undef CONF_MATCH
   1302 #undef CONF_MATCH_VALUE
   1303 #undef CONF_HANDLE_BOOL
   1304 #undef CONF_MIN_no
   1305 #undef CONF_MIN_yes
   1306 #undef CONF_MAX_no
   1307 #undef CONF_MAX_yes
   1308 #undef CONF_HANDLE_T_U
   1309 #undef CONF_HANDLE_UNSIGNED
   1310 #undef CONF_HANDLE_SIZE_T
   1311 #undef CONF_HANDLE_SSIZE_T
   1312 #undef CONF_HANDLE_CHAR_P
   1313 		}
   1314 	}
   1315 }
   1316 
   1317 static bool
   1318 malloc_init_hard_needed(void)
   1319 {
   1320 
   1321 	if (malloc_initialized() || (IS_INITIALIZER && malloc_init_state ==
   1322 	    malloc_init_recursible)) {
   1323 		/*
   1324 		 * Another thread initialized the allocator before this one
   1325 		 * acquired init_lock, or this thread is the initializing
   1326 		 * thread, and it is recursively allocating.
   1327 		 */
   1328 		return (false);
   1329 	}
   1330 #ifdef JEMALLOC_THREADED_INIT
   1331 	if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) {
   1332 		spin_t spinner;
   1333 
   1334 		/* Busy-wait until the initializing thread completes. */
   1335 		spin_init(&spinner);
   1336 		do {
   1337 			malloc_mutex_unlock(TSDN_NULL, &init_lock);
   1338 			spin_adaptive(&spinner);
   1339 			malloc_mutex_lock(TSDN_NULL, &init_lock);
   1340 		} while (!malloc_initialized());
   1341 		return (false);
   1342 	}
   1343 #endif
   1344 	return (true);
   1345 }
   1346 
   1347 static bool
   1348 malloc_init_hard_a0_locked()
   1349 {
   1350 
   1351 	malloc_initializer = INITIALIZER;
   1352 
   1353 	if (config_prof)
   1354 		prof_boot0();
   1355 	malloc_conf_init();
   1356 	if (opt_stats_print) {
   1357 		/* Print statistics at exit. */
   1358 		if (atexit(stats_print_atexit) != 0) {
   1359 			malloc_write("<jemalloc>: Error in atexit()\n");
   1360 			if (opt_abort)
   1361 				abort();
   1362 		}
   1363 	}
   1364 	pages_boot();
   1365 	if (base_boot())
   1366 		return (true);
   1367 	if (chunk_boot())
   1368 		return (true);
   1369 	if (ctl_boot())
   1370 		return (true);
   1371 	if (config_prof)
   1372 		prof_boot1();
   1373 	arena_boot();
   1374 	if (config_tcache && tcache_boot(TSDN_NULL))
   1375 		return (true);
   1376 	if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS))
   1377 		return (true);
   1378 	/*
   1379 	 * Create enough scaffolding to allow recursive allocation in
   1380 	 * malloc_ncpus().
   1381 	 */
   1382 	narenas_auto = 1;
   1383 	narenas_total_set(narenas_auto);
   1384 	arenas = &a0;
   1385 	memset(arenas, 0, sizeof(arena_t *) * narenas_auto);
   1386 	/*
   1387 	 * Initialize one arena here.  The rest are lazily created in
   1388 	 * arena_choose_hard().
   1389 	 */
   1390 	if (arena_init(TSDN_NULL, 0) == NULL)
   1391 		return (true);
   1392 
   1393 	malloc_init_state = malloc_init_a0_initialized;
   1394 
   1395 	return (false);
   1396 }
   1397 
   1398 static bool
   1399 malloc_init_hard_a0(void)
   1400 {
   1401 	bool ret;
   1402 
   1403 	malloc_mutex_lock(TSDN_NULL, &init_lock);
   1404 	ret = malloc_init_hard_a0_locked();
   1405 	malloc_mutex_unlock(TSDN_NULL, &init_lock);
   1406 	return (ret);
   1407 }
   1408 
   1409 /* Initialize data structures which may trigger recursive allocation. */
   1410 static bool
   1411 malloc_init_hard_recursible(void)
   1412 {
   1413 
   1414 	malloc_init_state = malloc_init_recursible;
   1415 
   1416 	ncpus = malloc_ncpus();
   1417 
   1418 #if (defined(JEMALLOC_HAVE_PTHREAD_ATFORK) && !defined(JEMALLOC_MUTEX_INIT_CB) \
   1419     && !defined(JEMALLOC_ZONE) && !defined(_WIN32) && \
   1420     !defined(__native_client__))
   1421 	/* LinuxThreads' pthread_atfork() allocates. */
   1422 	if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent,
   1423 	    jemalloc_postfork_child) != 0) {
   1424 		malloc_write("<jemalloc>: Error in pthread_atfork()\n");
   1425 		if (opt_abort)
   1426 			abort();
   1427 		return (true);
   1428 	}
   1429 #endif
   1430 
   1431 	return (false);
   1432 }
   1433 
   1434 static bool
   1435 malloc_init_hard_finish(tsdn_t *tsdn)
   1436 {
   1437 
   1438 	if (malloc_mutex_boot())
   1439 		return (true);
   1440 
   1441 	if (opt_narenas == 0) {
   1442 		/*
   1443 		 * For SMP systems, create more than one arena per CPU by
   1444 		 * default.
   1445 		 */
   1446 		if (ncpus > 1)
   1447 			opt_narenas = ncpus << 2;
   1448 		else
   1449 			opt_narenas = 1;
   1450 	}
   1451 #if defined(ANDROID_MAX_ARENAS)
   1452 	/* Never create more than MAX_ARENAS arenas regardless of num_cpus.
   1453 	 * Extra arenas use more PSS and are not very useful unless
   1454 	 * lots of threads are allocing/freeing at the same time.
   1455 	 */
   1456 	if (opt_narenas > ANDROID_MAX_ARENAS)
   1457 		opt_narenas = ANDROID_MAX_ARENAS;
   1458 #endif
   1459 	narenas_auto = opt_narenas;
   1460 	/*
   1461 	 * Limit the number of arenas to the indexing range of MALLOCX_ARENA().
   1462 	 */
   1463 	if (narenas_auto > MALLOCX_ARENA_MAX) {
   1464 		narenas_auto = MALLOCX_ARENA_MAX;
   1465 		malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n",
   1466 		    narenas_auto);
   1467 	}
   1468 	narenas_total_set(narenas_auto);
   1469 
   1470 	/* Allocate and initialize arenas. */
   1471 	arenas = (arena_t **)base_alloc(tsdn, sizeof(arena_t *) *
   1472 	    (MALLOCX_ARENA_MAX+1));
   1473 	if (arenas == NULL)
   1474 		return (true);
   1475 	/* Copy the pointer to the one arena that was already initialized. */
   1476 	arena_set(0, a0);
   1477 
   1478 	malloc_init_state = malloc_init_initialized;
   1479 	malloc_slow_flag_init();
   1480 
   1481 	return (false);
   1482 }
   1483 
   1484 static bool
   1485 malloc_init_hard(void)
   1486 {
   1487 	tsd_t *tsd;
   1488 
   1489 #if defined(_WIN32) && _WIN32_WINNT < 0x0600
   1490 	_init_init_lock();
   1491 #endif
   1492 	malloc_mutex_lock(TSDN_NULL, &init_lock);
   1493 	if (!malloc_init_hard_needed()) {
   1494 		malloc_mutex_unlock(TSDN_NULL, &init_lock);
   1495 		return (false);
   1496 	}
   1497 
   1498 	if (malloc_init_state != malloc_init_a0_initialized &&
   1499 	    malloc_init_hard_a0_locked()) {
   1500 		malloc_mutex_unlock(TSDN_NULL, &init_lock);
   1501 		return (true);
   1502 	}
   1503 
   1504 	malloc_mutex_unlock(TSDN_NULL, &init_lock);
   1505 	/* Recursive allocation relies on functional tsd. */
   1506 	tsd = malloc_tsd_boot0();
   1507 	if (tsd == NULL)
   1508 		return (true);
   1509 	if (malloc_init_hard_recursible())
   1510 		return (true);
   1511 	malloc_mutex_lock(tsd_tsdn(tsd), &init_lock);
   1512 
   1513 	if (config_prof && prof_boot2(tsd)) {
   1514 		malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);
   1515 		return (true);
   1516 	}
   1517 
   1518 	if (malloc_init_hard_finish(tsd_tsdn(tsd))) {
   1519 		malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);
   1520 		return (true);
   1521 	}
   1522 
   1523 	malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);
   1524 	malloc_tsd_boot1();
   1525 	return (false);
   1526 }
   1527 
   1528 /*
   1529  * End initialization functions.
   1530  */
   1531 /******************************************************************************/
   1532 /*
   1533  * Begin malloc(3)-compatible functions.
   1534  */
   1535 
   1536 static void *
   1537 ialloc_prof_sample(tsd_t *tsd, size_t usize, szind_t ind, bool zero,
   1538     prof_tctx_t *tctx, bool slow_path)
   1539 {
   1540 	void *p;
   1541 
   1542 	if (tctx == NULL)
   1543 		return (NULL);
   1544 	if (usize <= SMALL_MAXCLASS) {
   1545 		szind_t ind_large = size2index(LARGE_MINCLASS);
   1546 		p = ialloc(tsd, LARGE_MINCLASS, ind_large, zero, slow_path);
   1547 		if (p == NULL)
   1548 			return (NULL);
   1549 		arena_prof_promoted(tsd_tsdn(tsd), p, usize);
   1550 	} else
   1551 		p = ialloc(tsd, usize, ind, zero, slow_path);
   1552 
   1553 	return (p);
   1554 }
   1555 
   1556 JEMALLOC_ALWAYS_INLINE_C void *
   1557 ialloc_prof(tsd_t *tsd, size_t usize, szind_t ind, bool zero, bool slow_path)
   1558 {
   1559 	void *p;
   1560 	prof_tctx_t *tctx;
   1561 
   1562 	tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true);
   1563 	if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
   1564 		p = ialloc_prof_sample(tsd, usize, ind, zero, tctx, slow_path);
   1565 	else
   1566 		p = ialloc(tsd, usize, ind, zero, slow_path);
   1567 	if (unlikely(p == NULL)) {
   1568 		prof_alloc_rollback(tsd, tctx, true);
   1569 		return (NULL);
   1570 	}
   1571 	prof_malloc(tsd_tsdn(tsd), p, usize, tctx);
   1572 
   1573 	return (p);
   1574 }
   1575 
   1576 /*
   1577  * ialloc_body() is inlined so that fast and slow paths are generated separately
   1578  * with statically known slow_path.
   1579  *
   1580  * This function guarantees that *tsdn is non-NULL on success.
   1581  */
   1582 JEMALLOC_ALWAYS_INLINE_C void *
   1583 ialloc_body(size_t size, bool zero, tsdn_t **tsdn, size_t *usize,
   1584     bool slow_path)
   1585 {
   1586 	tsd_t *tsd;
   1587 	szind_t ind;
   1588 
   1589 	if (slow_path && unlikely(malloc_init())) {
   1590 		*tsdn = NULL;
   1591 		return (NULL);
   1592 	}
   1593 
   1594 	tsd = tsd_fetch();
   1595 	*tsdn = tsd_tsdn(tsd);
   1596 	witness_assert_lockless(tsd_tsdn(tsd));
   1597 
   1598 	ind = size2index(size);
   1599 	if (unlikely(ind >= NSIZES))
   1600 		return (NULL);
   1601 
   1602 	if (config_stats || (config_prof && opt_prof) || (slow_path &&
   1603 	    config_valgrind && unlikely(in_valgrind))) {
   1604 		*usize = index2size(ind);
   1605 		assert(*usize > 0 && *usize <= HUGE_MAXCLASS);
   1606 	}
   1607 
   1608 	if (config_prof && opt_prof)
   1609 		return (ialloc_prof(tsd, *usize, ind, zero, slow_path));
   1610 
   1611 	return (ialloc(tsd, size, ind, zero, slow_path));
   1612 }
   1613 
   1614 JEMALLOC_ALWAYS_INLINE_C void
   1615 ialloc_post_check(void *ret, tsdn_t *tsdn, size_t usize, const char *func,
   1616     bool update_errno, bool slow_path)
   1617 {
   1618 
   1619 	assert(!tsdn_null(tsdn) || ret == NULL);
   1620 
   1621 	if (unlikely(ret == NULL)) {
   1622 		if (slow_path && config_xmalloc && unlikely(opt_xmalloc)) {
   1623 			malloc_printf("<jemalloc>: Error in %s(): out of "
   1624 			    "memory\n", func);
   1625 			abort();
   1626 		}
   1627 		if (update_errno)
   1628 			set_errno(ENOMEM);
   1629 	}
   1630 	if (config_stats && likely(ret != NULL)) {
   1631 		assert(usize == isalloc(tsdn, ret, config_prof));
   1632 		*tsd_thread_allocatedp_get(tsdn_tsd(tsdn)) += usize;
   1633 	}
   1634 	witness_assert_lockless(tsdn);
   1635 }
   1636 
   1637 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
   1638 void JEMALLOC_NOTHROW *
   1639 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
   1640 je_malloc(size_t size)
   1641 {
   1642 	void *ret;
   1643 	tsdn_t *tsdn;
   1644 	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
   1645 
   1646 	if (size == 0)
   1647 		size = 1;
   1648 
   1649 	if (likely(!malloc_slow)) {
   1650 		ret = ialloc_body(size, false, &tsdn, &usize, false);
   1651 		ialloc_post_check(ret, tsdn, usize, "malloc", true, false);
   1652 	} else {
   1653 		ret = ialloc_body(size, false, &tsdn, &usize, true);
   1654 		ialloc_post_check(ret, tsdn, usize, "malloc", true, true);
   1655 		UTRACE(0, size, ret);
   1656 		JEMALLOC_VALGRIND_MALLOC(ret != NULL, tsdn, ret, usize, false);
   1657 	}
   1658 
   1659 	return (ret);
   1660 }
   1661 
   1662 static void *
   1663 imemalign_prof_sample(tsd_t *tsd, size_t alignment, size_t usize,
   1664     prof_tctx_t *tctx)
   1665 {
   1666 	void *p;
   1667 
   1668 	if (tctx == NULL)
   1669 		return (NULL);
   1670 	if (usize <= SMALL_MAXCLASS) {
   1671 		assert(sa2u(LARGE_MINCLASS, alignment) == LARGE_MINCLASS);
   1672 		p = ipalloc(tsd, LARGE_MINCLASS, alignment, false);
   1673 		if (p == NULL)
   1674 			return (NULL);
   1675 		arena_prof_promoted(tsd_tsdn(tsd), p, usize);
   1676 	} else
   1677 		p = ipalloc(tsd, usize, alignment, false);
   1678 
   1679 	return (p);
   1680 }
   1681 
   1682 JEMALLOC_ALWAYS_INLINE_C void *
   1683 imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize)
   1684 {
   1685 	void *p;
   1686 	prof_tctx_t *tctx;
   1687 
   1688 	tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true);
   1689 	if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
   1690 		p = imemalign_prof_sample(tsd, alignment, usize, tctx);
   1691 	else
   1692 		p = ipalloc(tsd, usize, alignment, false);
   1693 	if (unlikely(p == NULL)) {
   1694 		prof_alloc_rollback(tsd, tctx, true);
   1695 		return (NULL);
   1696 	}
   1697 	prof_malloc(tsd_tsdn(tsd), p, usize, tctx);
   1698 
   1699 	return (p);
   1700 }
   1701 
   1702 JEMALLOC_ATTR(nonnull(1))
   1703 static int
   1704 imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
   1705 {
   1706 	int ret;
   1707 	tsd_t *tsd;
   1708 	size_t usize;
   1709 	void *result;
   1710 
   1711 	assert(min_alignment != 0);
   1712 
   1713 	if (unlikely(malloc_init())) {
   1714 		tsd = NULL;
   1715 		result = NULL;
   1716 		goto label_oom;
   1717 	}
   1718 	tsd = tsd_fetch();
   1719 	witness_assert_lockless(tsd_tsdn(tsd));
   1720 	if (size == 0)
   1721 		size = 1;
   1722 
   1723 	/* Make sure that alignment is a large enough power of 2. */
   1724 	if (unlikely(((alignment - 1) & alignment) != 0
   1725 	    || (alignment < min_alignment))) {
   1726 		if (config_xmalloc && unlikely(opt_xmalloc)) {
   1727 			malloc_write("<jemalloc>: Error allocating "
   1728 			    "aligned memory: invalid alignment\n");
   1729 			abort();
   1730 		}
   1731 		result = NULL;
   1732 		ret = EINVAL;
   1733 		goto label_return;
   1734 	}
   1735 
   1736 	usize = sa2u(size, alignment);
   1737 	if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) {
   1738 		result = NULL;
   1739 		goto label_oom;
   1740 	}
   1741 
   1742 	if (config_prof && opt_prof)
   1743 		result = imemalign_prof(tsd, alignment, usize);
   1744 	else
   1745 		result = ipalloc(tsd, usize, alignment, false);
   1746 	if (unlikely(result == NULL))
   1747 		goto label_oom;
   1748 	assert(((uintptr_t)result & (alignment - 1)) == ZU(0));
   1749 
   1750 	*memptr = result;
   1751 	ret = 0;
   1752 label_return:
   1753 	if (config_stats && likely(result != NULL)) {
   1754 		assert(usize == isalloc(tsd_tsdn(tsd), result, config_prof));
   1755 		*tsd_thread_allocatedp_get(tsd) += usize;
   1756 	}
   1757 	UTRACE(0, size, result);
   1758 	JEMALLOC_VALGRIND_MALLOC(result != NULL, tsd_tsdn(tsd), result, usize,
   1759 	    false);
   1760 	witness_assert_lockless(tsd_tsdn(tsd));
   1761 	return (ret);
   1762 label_oom:
   1763 	assert(result == NULL);
   1764 	if (config_xmalloc && unlikely(opt_xmalloc)) {
   1765 		malloc_write("<jemalloc>: Error allocating aligned memory: "
   1766 		    "out of memory\n");
   1767 		abort();
   1768 	}
   1769 	ret = ENOMEM;
   1770 	witness_assert_lockless(tsd_tsdn(tsd));
   1771 	goto label_return;
   1772 }
   1773 
   1774 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
   1775 JEMALLOC_ATTR(nonnull(1))
   1776 je_posix_memalign(void **memptr, size_t alignment, size_t size)
   1777 {
   1778 	int ret;
   1779 
   1780 	ret = imemalign(memptr, alignment, size, sizeof(void *));
   1781 
   1782 	return (ret);
   1783 }
   1784 
   1785 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
   1786 void JEMALLOC_NOTHROW *
   1787 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(2)
   1788 je_aligned_alloc(size_t alignment, size_t size)
   1789 {
   1790 	void *ret;
   1791 	int err;
   1792 
   1793 	if (unlikely((err = imemalign(&ret, alignment, size, 1)) != 0)) {
   1794 		ret = NULL;
   1795 		set_errno(err);
   1796 	}
   1797 
   1798 	return (ret);
   1799 }
   1800 
   1801 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
   1802 void JEMALLOC_NOTHROW *
   1803 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2)
   1804 je_calloc(size_t num, size_t size)
   1805 {
   1806 	void *ret;
   1807 	tsdn_t *tsdn;
   1808 	size_t num_size;
   1809 	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
   1810 
   1811 	num_size = num * size;
   1812 	if (unlikely(num_size == 0)) {
   1813 		if (num == 0 || size == 0)
   1814 			num_size = 1;
   1815 		else
   1816 			num_size = HUGE_MAXCLASS + 1; /* Trigger OOM. */
   1817 	/*
   1818 	 * Try to avoid division here.  We know that it isn't possible to
   1819 	 * overflow during multiplication if neither operand uses any of the
   1820 	 * most significant half of the bits in a size_t.
   1821 	 */
   1822 	} else if (unlikely(((num | size) & (SIZE_T_MAX << (sizeof(size_t) <<
   1823 	    2))) && (num_size / size != num)))
   1824 		num_size = HUGE_MAXCLASS + 1; /* size_t overflow. */
   1825 
   1826 	if (likely(!malloc_slow)) {
   1827 		ret = ialloc_body(num_size, true, &tsdn, &usize, false);
   1828 		ialloc_post_check(ret, tsdn, usize, "calloc", true, false);
   1829 	} else {
   1830 		ret = ialloc_body(num_size, true, &tsdn, &usize, true);
   1831 		ialloc_post_check(ret, tsdn, usize, "calloc", true, true);
   1832 		UTRACE(0, num_size, ret);
   1833 		JEMALLOC_VALGRIND_MALLOC(ret != NULL, tsdn, ret, usize, true);
   1834 	}
   1835 
   1836 	return (ret);
   1837 }
   1838 
   1839 static void *
   1840 irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize,
   1841     prof_tctx_t *tctx)
   1842 {
   1843 	void *p;
   1844 
   1845 	if (tctx == NULL)
   1846 		return (NULL);
   1847 	if (usize <= SMALL_MAXCLASS) {
   1848 		p = iralloc(tsd, old_ptr, old_usize, LARGE_MINCLASS, 0, false);
   1849 		if (p == NULL)
   1850 			return (NULL);
   1851 		arena_prof_promoted(tsd_tsdn(tsd), p, usize);
   1852 	} else
   1853 		p = iralloc(tsd, old_ptr, old_usize, usize, 0, false);
   1854 
   1855 	return (p);
   1856 }
   1857 
   1858 JEMALLOC_ALWAYS_INLINE_C void *
   1859 irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize)
   1860 {
   1861 	void *p;
   1862 	bool prof_active;
   1863 	prof_tctx_t *old_tctx, *tctx;
   1864 
   1865 	prof_active = prof_active_get_unlocked();
   1866 	old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr);
   1867 	tctx = prof_alloc_prep(tsd, usize, prof_active, true);
   1868 	if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
   1869 		p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx);
   1870 	else
   1871 		p = iralloc(tsd, old_ptr, old_usize, usize, 0, false);
   1872 	if (unlikely(p == NULL)) {
   1873 		prof_alloc_rollback(tsd, tctx, true);
   1874 		return (NULL);
   1875 	}
   1876 	prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize,
   1877 	    old_tctx);
   1878 
   1879 	return (p);
   1880 }
   1881 
   1882 JEMALLOC_INLINE_C void
   1883 ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path)
   1884 {
   1885 	size_t usize;
   1886 	UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
   1887 
   1888 	witness_assert_lockless(tsd_tsdn(tsd));
   1889 
   1890 	assert(ptr != NULL);
   1891 	assert(malloc_initialized() || IS_INITIALIZER);
   1892 
   1893 	if (config_prof && opt_prof) {
   1894 		usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
   1895 		prof_free(tsd, ptr, usize);
   1896 	} else if (config_stats || config_valgrind)
   1897 		usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
   1898 	if (config_stats)
   1899 		*tsd_thread_deallocatedp_get(tsd) += usize;
   1900 
   1901 	if (likely(!slow_path))
   1902 		iqalloc(tsd, ptr, tcache, false);
   1903 	else {
   1904 		if (config_valgrind && unlikely(in_valgrind))
   1905 			rzsize = p2rz(tsd_tsdn(tsd), ptr);
   1906 		iqalloc(tsd, ptr, tcache, true);
   1907 		JEMALLOC_VALGRIND_FREE(ptr, rzsize);
   1908 	}
   1909 }
   1910 
   1911 JEMALLOC_INLINE_C void
   1912 isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path)
   1913 {
   1914 	UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
   1915 
   1916 	witness_assert_lockless(tsd_tsdn(tsd));
   1917 
   1918 	assert(ptr != NULL);
   1919 	assert(malloc_initialized() || IS_INITIALIZER);
   1920 
   1921 	if (config_prof && opt_prof)
   1922 		prof_free(tsd, ptr, usize);
   1923 	if (config_stats)
   1924 		*tsd_thread_deallocatedp_get(tsd) += usize;
   1925 	if (config_valgrind && unlikely(in_valgrind))
   1926 		rzsize = p2rz(tsd_tsdn(tsd), ptr);
   1927 	isqalloc(tsd, ptr, usize, tcache, slow_path);
   1928 	JEMALLOC_VALGRIND_FREE(ptr, rzsize);
   1929 }
   1930 
   1931 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
   1932 void JEMALLOC_NOTHROW *
   1933 JEMALLOC_ALLOC_SIZE(2)
   1934 je_realloc(void *ptr, size_t size)
   1935 {
   1936 	void *ret;
   1937 	tsdn_t *tsdn JEMALLOC_CC_SILENCE_INIT(NULL);
   1938 	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
   1939 	size_t old_usize = 0;
   1940 	UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
   1941 
   1942 	if (unlikely(size == 0)) {
   1943 		if (ptr != NULL) {
   1944 			tsd_t *tsd;
   1945 
   1946 			/* realloc(ptr, 0) is equivalent to free(ptr). */
   1947 			UTRACE(ptr, 0, 0);
   1948 			tsd = tsd_fetch();
   1949 			ifree(tsd, ptr, tcache_get(tsd, false), true);
   1950 			return (NULL);
   1951 		}
   1952 		size = 1;
   1953 	}
   1954 
   1955 	if (likely(ptr != NULL)) {
   1956 		tsd_t *tsd;
   1957 
   1958 		assert(malloc_initialized() || IS_INITIALIZER);
   1959 		malloc_thread_init();
   1960 		tsd = tsd_fetch();
   1961 
   1962 		witness_assert_lockless(tsd_tsdn(tsd));
   1963 
   1964 		old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
   1965 		if (config_valgrind && unlikely(in_valgrind)) {
   1966 			old_rzsize = config_prof ? p2rz(tsd_tsdn(tsd), ptr) :
   1967 			    u2rz(old_usize);
   1968 		}
   1969 
   1970 		if (config_prof && opt_prof) {
   1971 			usize = s2u(size);
   1972 			ret = unlikely(usize == 0 || usize > HUGE_MAXCLASS) ?
   1973 			    NULL : irealloc_prof(tsd, ptr, old_usize, usize);
   1974 		} else {
   1975 			if (config_stats || (config_valgrind &&
   1976 			    unlikely(in_valgrind)))
   1977 				usize = s2u(size);
   1978 			ret = iralloc(tsd, ptr, old_usize, size, 0, false);
   1979 		}
   1980 		tsdn = tsd_tsdn(tsd);
   1981 	} else {
   1982 		/* realloc(NULL, size) is equivalent to malloc(size). */
   1983 		if (likely(!malloc_slow))
   1984 			ret = ialloc_body(size, false, &tsdn, &usize, false);
   1985 		else
   1986 			ret = ialloc_body(size, false, &tsdn, &usize, true);
   1987 		assert(!tsdn_null(tsdn) || ret == NULL);
   1988 	}
   1989 
   1990 	if (unlikely(ret == NULL)) {
   1991 		if (config_xmalloc && unlikely(opt_xmalloc)) {
   1992 			malloc_write("<jemalloc>: Error in realloc(): "
   1993 			    "out of memory\n");
   1994 			abort();
   1995 		}
   1996 		set_errno(ENOMEM);
   1997 	}
   1998 	if (config_stats && likely(ret != NULL)) {
   1999 		tsd_t *tsd;
   2000 
   2001 		assert(usize == isalloc(tsdn, ret, config_prof));
   2002 		tsd = tsdn_tsd(tsdn);
   2003 		*tsd_thread_allocatedp_get(tsd) += usize;
   2004 		*tsd_thread_deallocatedp_get(tsd) += old_usize;
   2005 	}
   2006 	UTRACE(ptr, size, ret);
   2007 	JEMALLOC_VALGRIND_REALLOC(maybe, tsdn, ret, usize, maybe, ptr,
   2008 	    old_usize, old_rzsize, maybe, false);
   2009 	witness_assert_lockless(tsdn);
   2010 	return (ret);
   2011 }
   2012 
   2013 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
   2014 je_free(void *ptr)
   2015 {
   2016 
   2017 	UTRACE(ptr, 0, 0);
   2018 	if (likely(ptr != NULL)) {
   2019 		tsd_t *tsd = tsd_fetch();
   2020 		witness_assert_lockless(tsd_tsdn(tsd));
   2021 		if (likely(!malloc_slow))
   2022 			ifree(tsd, ptr, tcache_get(tsd, false), false);
   2023 		else
   2024 			ifree(tsd, ptr, tcache_get(tsd, false), true);
   2025 		witness_assert_lockless(tsd_tsdn(tsd));
   2026 	}
   2027 }
   2028 
   2029 /*
   2030  * End malloc(3)-compatible functions.
   2031  */
   2032 /******************************************************************************/
   2033 /*
   2034  * Begin non-standard override functions.
   2035  */
   2036 
   2037 #ifdef JEMALLOC_OVERRIDE_MEMALIGN
   2038 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
   2039 void JEMALLOC_NOTHROW *
   2040 JEMALLOC_ATTR(malloc)
   2041 je_memalign(size_t alignment, size_t size)
   2042 {
   2043 	void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
   2044 	if (unlikely(imemalign(&ret, alignment, size, 1) != 0))
   2045 		ret = NULL;
   2046 	return (ret);
   2047 }
   2048 #endif
   2049 
   2050 #ifdef JEMALLOC_OVERRIDE_VALLOC
   2051 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
   2052 void JEMALLOC_NOTHROW *
   2053 JEMALLOC_ATTR(malloc)
   2054 je_valloc(size_t size)
   2055 {
   2056 	void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
   2057 	if (unlikely(imemalign(&ret, PAGE, size, 1) != 0))
   2058 		ret = NULL;
   2059 	return (ret);
   2060 }
   2061 #endif
   2062 
   2063 /*
   2064  * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has
   2065  * #define je_malloc malloc
   2066  */
   2067 #define	malloc_is_malloc 1
   2068 #define	is_malloc_(a) malloc_is_ ## a
   2069 #define	is_malloc(a) is_malloc_(a)
   2070 
   2071 #if ((is_malloc(je_malloc) == 1) && defined(JEMALLOC_GLIBC_MALLOC_HOOK))
   2072 /*
   2073  * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
   2074  * to inconsistently reference libc's malloc(3)-compatible functions
   2075  * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541).
   2076  *
   2077  * These definitions interpose hooks in glibc.  The functions are actually
   2078  * passed an extra argument for the caller return address, which will be
   2079  * ignored.
   2080  */
   2081 JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free;
   2082 JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc;
   2083 JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc;
   2084 # ifdef JEMALLOC_GLIBC_MEMALIGN_HOOK
   2085 JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) =
   2086     je_memalign;
   2087 # endif
   2088 
   2089 #ifdef CPU_COUNT
   2090 /*
   2091  * To enable static linking with glibc, the libc specific malloc interface must
   2092  * be implemented also, so none of glibc's malloc.o functions are added to the
   2093  * link.
   2094  */
   2095 #define	ALIAS(je_fn)	__attribute__((alias (#je_fn), used))
   2096 /* To force macro expansion of je_ prefix before stringification. */
   2097 #define	PREALIAS(je_fn)  ALIAS(je_fn)
   2098 void	*__libc_malloc(size_t size) PREALIAS(je_malloc);
   2099 void	__libc_free(void* ptr) PREALIAS(je_free);
   2100 void	*__libc_realloc(void* ptr, size_t size) PREALIAS(je_realloc);
   2101 void	*__libc_calloc(size_t n, size_t size) PREALIAS(je_calloc);
   2102 void	*__libc_memalign(size_t align, size_t s) PREALIAS(je_memalign);
   2103 void	*__libc_valloc(size_t size) PREALIAS(je_valloc);
   2104 int	__posix_memalign(void** r, size_t a, size_t s)
   2105     PREALIAS(je_posix_memalign);
   2106 #undef PREALIAS
   2107 #undef ALIAS
   2108 
   2109 #endif
   2110 
   2111 #endif
   2112 
   2113 /*
   2114  * End non-standard override functions.
   2115  */
   2116 /******************************************************************************/
   2117 /*
   2118  * Begin non-standard functions.
   2119  */
   2120 
   2121 JEMALLOC_ALWAYS_INLINE_C bool
   2122 imallocx_flags_decode(tsd_t *tsd, size_t size, int flags, size_t *usize,
   2123     size_t *alignment, bool *zero, tcache_t **tcache, arena_t **arena)
   2124 {
   2125 
   2126 	if ((flags & MALLOCX_LG_ALIGN_MASK) == 0) {
   2127 		*alignment = 0;
   2128 		*usize = s2u(size);
   2129 	} else {
   2130 		*alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags);
   2131 		*usize = sa2u(size, *alignment);
   2132 	}
   2133 	if (unlikely(*usize == 0 || *usize > HUGE_MAXCLASS))
   2134 		return (true);
   2135 	*zero = MALLOCX_ZERO_GET(flags);
   2136 	if ((flags & MALLOCX_TCACHE_MASK) != 0) {
   2137 		if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE)
   2138 			*tcache = NULL;
   2139 		else
   2140 			*tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
   2141 	} else
   2142 		*tcache = tcache_get(tsd, true);
   2143 	if ((flags & MALLOCX_ARENA_MASK) != 0) {
   2144 		unsigned arena_ind = MALLOCX_ARENA_GET(flags);
   2145 		*arena = arena_get(tsd_tsdn(tsd), arena_ind, true);
   2146 		if (unlikely(*arena == NULL))
   2147 			return (true);
   2148 	} else
   2149 		*arena = NULL;
   2150 	return (false);
   2151 }
   2152 
   2153 JEMALLOC_ALWAYS_INLINE_C void *
   2154 imallocx_flags(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
   2155     tcache_t *tcache, arena_t *arena, bool slow_path)
   2156 {
   2157 	szind_t ind;
   2158 
   2159 	if (unlikely(alignment != 0))
   2160 		return (ipalloct(tsdn, usize, alignment, zero, tcache, arena));
   2161 	ind = size2index(usize);
   2162 	assert(ind < NSIZES);
   2163 	return (iallocztm(tsdn, usize, ind, zero, tcache, false, arena,
   2164 	    slow_path));
   2165 }
   2166 
   2167 static void *
   2168 imallocx_prof_sample(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
   2169     tcache_t *tcache, arena_t *arena, bool slow_path)
   2170 {
   2171 	void *p;
   2172 
   2173 	if (usize <= SMALL_MAXCLASS) {
   2174 		assert(((alignment == 0) ? s2u(LARGE_MINCLASS) :
   2175 		    sa2u(LARGE_MINCLASS, alignment)) == LARGE_MINCLASS);
   2176 		p = imallocx_flags(tsdn, LARGE_MINCLASS, alignment, zero,
   2177 		    tcache, arena, slow_path);
   2178 		if (p == NULL)
   2179 			return (NULL);
   2180 		arena_prof_promoted(tsdn, p, usize);
   2181 	} else {
   2182 		p = imallocx_flags(tsdn, usize, alignment, zero, tcache, arena,
   2183 		    slow_path);
   2184 	}
   2185 
   2186 	return (p);
   2187 }
   2188 
   2189 JEMALLOC_ALWAYS_INLINE_C void *
   2190 imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize, bool slow_path)
   2191 {
   2192 	void *p;
   2193 	size_t alignment;
   2194 	bool zero;
   2195 	tcache_t *tcache;
   2196 	arena_t *arena;
   2197 	prof_tctx_t *tctx;
   2198 
   2199 	if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment,
   2200 	    &zero, &tcache, &arena)))
   2201 		return (NULL);
   2202 	tctx = prof_alloc_prep(tsd, *usize, prof_active_get_unlocked(), true);
   2203 	if (likely((uintptr_t)tctx == (uintptr_t)1U)) {
   2204 		p = imallocx_flags(tsd_tsdn(tsd), *usize, alignment, zero,
   2205 		    tcache, arena, slow_path);
   2206 	} else if ((uintptr_t)tctx > (uintptr_t)1U) {
   2207 		p = imallocx_prof_sample(tsd_tsdn(tsd), *usize, alignment, zero,
   2208 		    tcache, arena, slow_path);
   2209 	} else
   2210 		p = NULL;
   2211 	if (unlikely(p == NULL)) {
   2212 		prof_alloc_rollback(tsd, tctx, true);
   2213 		return (NULL);
   2214 	}
   2215 	prof_malloc(tsd_tsdn(tsd), p, *usize, tctx);
   2216 
   2217 	assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
   2218 	return (p);
   2219 }
   2220 
   2221 JEMALLOC_ALWAYS_INLINE_C void *
   2222 imallocx_no_prof(tsd_t *tsd, size_t size, int flags, size_t *usize,
   2223     bool slow_path)
   2224 {
   2225 	void *p;
   2226 	size_t alignment;
   2227 	bool zero;
   2228 	tcache_t *tcache;
   2229 	arena_t *arena;
   2230 
   2231 	if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment,
   2232 	    &zero, &tcache, &arena)))
   2233 		return (NULL);
   2234 	p = imallocx_flags(tsd_tsdn(tsd), *usize, alignment, zero, tcache,
   2235 	    arena, slow_path);
   2236 	assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
   2237 	return (p);
   2238 }
   2239 
   2240 /* This function guarantees that *tsdn is non-NULL on success. */
   2241 JEMALLOC_ALWAYS_INLINE_C void *
   2242 imallocx_body(size_t size, int flags, tsdn_t **tsdn, size_t *usize,
   2243     bool slow_path)
   2244 {
   2245 	tsd_t *tsd;
   2246 
   2247 	if (slow_path && unlikely(malloc_init())) {
   2248 		*tsdn = NULL;
   2249 		return (NULL);
   2250 	}
   2251 
   2252 	tsd = tsd_fetch();
   2253 	*tsdn = tsd_tsdn(tsd);
   2254 	witness_assert_lockless(tsd_tsdn(tsd));
   2255 
   2256 	if (likely(flags == 0)) {
   2257 		szind_t ind = size2index(size);
   2258 		if (unlikely(ind >= NSIZES))
   2259 			return (NULL);
   2260 		if (config_stats || (config_prof && opt_prof) || (slow_path &&
   2261 		    config_valgrind && unlikely(in_valgrind))) {
   2262 			*usize = index2size(ind);
   2263 			assert(*usize > 0 && *usize <= HUGE_MAXCLASS);
   2264 		}
   2265 
   2266 		if (config_prof && opt_prof) {
   2267 			return (ialloc_prof(tsd, *usize, ind, false,
   2268 			    slow_path));
   2269 		}
   2270 
   2271 		return (ialloc(tsd, size, ind, false, slow_path));
   2272 	}
   2273 
   2274 	if (config_prof && opt_prof)
   2275 		return (imallocx_prof(tsd, size, flags, usize, slow_path));
   2276 
   2277 	return (imallocx_no_prof(tsd, size, flags, usize, slow_path));
   2278 }
   2279 
   2280 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
   2281 void JEMALLOC_NOTHROW *
   2282 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
   2283 je_mallocx(size_t size, int flags)
   2284 {
   2285 	tsdn_t *tsdn;
   2286 	void *p;
   2287 	size_t usize;
   2288 
   2289 	assert(size != 0);
   2290 
   2291 	if (likely(!malloc_slow)) {
   2292 		p = imallocx_body(size, flags, &tsdn, &usize, false);
   2293 		ialloc_post_check(p, tsdn, usize, "mallocx", false, false);
   2294 	} else {
   2295 		p = imallocx_body(size, flags, &tsdn, &usize, true);
   2296 		ialloc_post_check(p, tsdn, usize, "mallocx", false, true);
   2297 		UTRACE(0, size, p);
   2298 		JEMALLOC_VALGRIND_MALLOC(p != NULL, tsdn, p, usize,
   2299 		    MALLOCX_ZERO_GET(flags));
   2300 	}
   2301 
   2302 	return (p);
   2303 }
   2304 
   2305 static void *
   2306 irallocx_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize,
   2307     size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena,
   2308     prof_tctx_t *tctx)
   2309 {
   2310 	void *p;
   2311 
   2312 	if (tctx == NULL)
   2313 		return (NULL);
   2314 	if (usize <= SMALL_MAXCLASS) {
   2315 		p = iralloct(tsd, old_ptr, old_usize, LARGE_MINCLASS, alignment,
   2316 		    zero, tcache, arena);
   2317 		if (p == NULL)
   2318 			return (NULL);
   2319 		arena_prof_promoted(tsd_tsdn(tsd), p, usize);
   2320 	} else {
   2321 		p = iralloct(tsd, old_ptr, old_usize, usize, alignment, zero,
   2322 		    tcache, arena);
   2323 	}
   2324 
   2325 	return (p);
   2326 }
   2327 
   2328 JEMALLOC_ALWAYS_INLINE_C void *
   2329 irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
   2330     size_t alignment, size_t *usize, bool zero, tcache_t *tcache,
   2331     arena_t *arena)
   2332 {
   2333 	void *p;
   2334 	bool prof_active;
   2335 	prof_tctx_t *old_tctx, *tctx;
   2336 
   2337 	prof_active = prof_active_get_unlocked();
   2338 	old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr);
   2339 	tctx = prof_alloc_prep(tsd, *usize, prof_active, false);
   2340 	if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
   2341 		p = irallocx_prof_sample(tsd, old_ptr, old_usize, *usize,
   2342 		    alignment, zero, tcache, arena, tctx);
   2343 	} else {
   2344 		p = iralloct(tsd, old_ptr, old_usize, size, alignment, zero,
   2345 		    tcache, arena);
   2346 	}
   2347 	if (unlikely(p == NULL)) {
   2348 		prof_alloc_rollback(tsd, tctx, false);
   2349 		return (NULL);
   2350 	}
   2351 
   2352 	if (p == old_ptr && alignment != 0) {
   2353 		/*
   2354 		 * The allocation did not move, so it is possible that the size
   2355 		 * class is smaller than would guarantee the requested
   2356 		 * alignment, and that the alignment constraint was
   2357 		 * serendipitously satisfied.  Additionally, old_usize may not
   2358 		 * be the same as the current usize because of in-place large
   2359 		 * reallocation.  Therefore, query the actual value of usize.
   2360 		 */
   2361 		*usize = isalloc(tsd_tsdn(tsd), p, config_prof);
   2362 	}
   2363 	prof_realloc(tsd, p, *usize, tctx, prof_active, false, old_ptr,
   2364 	    old_usize, old_tctx);
   2365 
   2366 	return (p);
   2367 }
   2368 
   2369 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
   2370 void JEMALLOC_NOTHROW *
   2371 JEMALLOC_ALLOC_SIZE(2)
   2372 je_rallocx(void *ptr, size_t size, int flags)
   2373 {
   2374 	void *p;
   2375 	tsd_t *tsd;
   2376 	size_t usize;
   2377 	size_t old_usize;
   2378 	UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
   2379 	size_t alignment = MALLOCX_ALIGN_GET(flags);
   2380 	bool zero = flags & MALLOCX_ZERO;
   2381 	arena_t *arena;
   2382 	tcache_t *tcache;
   2383 
   2384 	assert(ptr != NULL);
   2385 	assert(size != 0);
   2386 	assert(malloc_initialized() || IS_INITIALIZER);
   2387 	malloc_thread_init();
   2388 	tsd = tsd_fetch();
   2389 	witness_assert_lockless(tsd_tsdn(tsd));
   2390 
   2391 	if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
   2392 		unsigned arena_ind = MALLOCX_ARENA_GET(flags);
   2393 		arena = arena_get(tsd_tsdn(tsd), arena_ind, true);
   2394 		if (unlikely(arena == NULL))
   2395 			goto label_oom;
   2396 	} else
   2397 		arena = NULL;
   2398 
   2399 	if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
   2400 		if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE)
   2401 			tcache = NULL;
   2402 		else
   2403 			tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
   2404 	} else
   2405 		tcache = tcache_get(tsd, true);
   2406 
   2407 	old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
   2408 	if (config_valgrind && unlikely(in_valgrind))
   2409 		old_rzsize = u2rz(old_usize);
   2410 
   2411 	if (config_prof && opt_prof) {
   2412 		usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
   2413 		if (unlikely(usize == 0 || usize > HUGE_MAXCLASS))
   2414 			goto label_oom;
   2415 		p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
   2416 		    zero, tcache, arena);
   2417 		if (unlikely(p == NULL))
   2418 			goto label_oom;
   2419 	} else {
   2420 		p = iralloct(tsd, ptr, old_usize, size, alignment, zero,
   2421 		     tcache, arena);
   2422 		if (unlikely(p == NULL))
   2423 			goto label_oom;
   2424 		if (config_stats || (config_valgrind && unlikely(in_valgrind)))
   2425 			usize = isalloc(tsd_tsdn(tsd), p, config_prof);
   2426 	}
   2427 	assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
   2428 
   2429 	if (config_stats) {
   2430 		*tsd_thread_allocatedp_get(tsd) += usize;
   2431 		*tsd_thread_deallocatedp_get(tsd) += old_usize;
   2432 	}
   2433 	UTRACE(ptr, size, p);
   2434 	JEMALLOC_VALGRIND_REALLOC(maybe, tsd_tsdn(tsd), p, usize, no, ptr,
   2435 	    old_usize, old_rzsize, no, zero);
   2436 	witness_assert_lockless(tsd_tsdn(tsd));
   2437 	return (p);
   2438 label_oom:
   2439 	if (config_xmalloc && unlikely(opt_xmalloc)) {
   2440 		malloc_write("<jemalloc>: Error in rallocx(): out of memory\n");
   2441 		abort();
   2442 	}
   2443 	UTRACE(ptr, size, 0);
   2444 	witness_assert_lockless(tsd_tsdn(tsd));
   2445 	return (NULL);
   2446 }
   2447 
   2448 JEMALLOC_ALWAYS_INLINE_C size_t
   2449 ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
   2450     size_t extra, size_t alignment, bool zero)
   2451 {
   2452 	size_t usize;
   2453 
   2454 	if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero))
   2455 		return (old_usize);
   2456 	usize = isalloc(tsdn, ptr, config_prof);
   2457 
   2458 	return (usize);
   2459 }
   2460 
   2461 static size_t
   2462 ixallocx_prof_sample(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
   2463     size_t extra, size_t alignment, bool zero, prof_tctx_t *tctx)
   2464 {
   2465 	size_t usize;
   2466 
   2467 	if (tctx == NULL)
   2468 		return (old_usize);
   2469 	usize = ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment,
   2470 	    zero);
   2471 
   2472 	return (usize);
   2473 }
   2474 
   2475 JEMALLOC_ALWAYS_INLINE_C size_t
   2476 ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
   2477     size_t extra, size_t alignment, bool zero)
   2478 {
   2479 	size_t usize_max, usize;
   2480 	bool prof_active;
   2481 	prof_tctx_t *old_tctx, *tctx;
   2482 
   2483 	prof_active = prof_active_get_unlocked();
   2484 	old_tctx = prof_tctx_get(tsd_tsdn(tsd), ptr);
   2485 	/*
   2486 	 * usize isn't knowable before ixalloc() returns when extra is non-zero.
   2487 	 * Therefore, compute its maximum possible value and use that in
   2488 	 * prof_alloc_prep() to decide whether to capture a backtrace.
   2489 	 * prof_realloc() will use the actual usize to decide whether to sample.
   2490 	 */
   2491 	if (alignment == 0) {
   2492 		usize_max = s2u(size+extra);
   2493 		assert(usize_max > 0 && usize_max <= HUGE_MAXCLASS);
   2494 	} else {
   2495 		usize_max = sa2u(size+extra, alignment);
   2496 		if (unlikely(usize_max == 0 || usize_max > HUGE_MAXCLASS)) {
   2497 			/*
   2498 			 * usize_max is out of range, and chances are that
   2499 			 * allocation will fail, but use the maximum possible
   2500 			 * value and carry on with prof_alloc_prep(), just in
   2501 			 * case allocation succeeds.
   2502 			 */
   2503 			usize_max = HUGE_MAXCLASS;
   2504 		}
   2505 	}
   2506 	tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
   2507 
   2508 	if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
   2509 		usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize,
   2510 		    size, extra, alignment, zero, tctx);
   2511 	} else {
   2512 		usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
   2513 		    extra, alignment, zero);
   2514 	}
   2515 	if (usize == old_usize) {
   2516 		prof_alloc_rollback(tsd, tctx, false);
   2517 		return (usize);
   2518 	}
   2519 	prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize,
   2520 	    old_tctx);
   2521 
   2522 	return (usize);
   2523 }
   2524 
   2525 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
   2526 je_xallocx(void *ptr, size_t size, size_t extra, int flags)
   2527 {
   2528 	tsd_t *tsd;
   2529 	size_t usize, old_usize;
   2530 	UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
   2531 	size_t alignment = MALLOCX_ALIGN_GET(flags);
   2532 	bool zero = flags & MALLOCX_ZERO;
   2533 
   2534 	assert(ptr != NULL);
   2535 	assert(size != 0);
   2536 	assert(SIZE_T_MAX - size >= extra);
   2537 	assert(malloc_initialized() || IS_INITIALIZER);
   2538 	malloc_thread_init();
   2539 	tsd = tsd_fetch();
   2540 	witness_assert_lockless(tsd_tsdn(tsd));
   2541 
   2542 	old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
   2543 
   2544 	/*
   2545 	 * The API explicitly absolves itself of protecting against (size +
   2546 	 * extra) numerical overflow, but we may need to clamp extra to avoid
   2547 	 * exceeding HUGE_MAXCLASS.
   2548 	 *
   2549 	 * Ordinarily, size limit checking is handled deeper down, but here we
   2550 	 * have to check as part of (size + extra) clamping, since we need the
   2551 	 * clamped value in the above helper functions.
   2552 	 */
   2553 	if (unlikely(size > HUGE_MAXCLASS)) {
   2554 		usize = old_usize;
   2555 		goto label_not_resized;
   2556 	}
   2557 	if (unlikely(HUGE_MAXCLASS - size < extra))
   2558 		extra = HUGE_MAXCLASS - size;
   2559 
   2560 	if (config_valgrind && unlikely(in_valgrind))
   2561 		old_rzsize = u2rz(old_usize);
   2562 
   2563 	if (config_prof && opt_prof) {
   2564 		usize = ixallocx_prof(tsd, ptr, old_usize, size, extra,
   2565 		    alignment, zero);
   2566 	} else {
   2567 		usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
   2568 		    extra, alignment, zero);
   2569 	}
   2570 	if (unlikely(usize == old_usize))
   2571 		goto label_not_resized;
   2572 
   2573 	if (config_stats) {
   2574 		*tsd_thread_allocatedp_get(tsd) += usize;
   2575 		*tsd_thread_deallocatedp_get(tsd) += old_usize;
   2576 	}
   2577 	JEMALLOC_VALGRIND_REALLOC(no, tsd_tsdn(tsd), ptr, usize, no, ptr,
   2578 	    old_usize, old_rzsize, no, zero);
   2579 label_not_resized:
   2580 	UTRACE(ptr, size, ptr);
   2581 	witness_assert_lockless(tsd_tsdn(tsd));
   2582 	return (usize);
   2583 }
   2584 
   2585 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
   2586 JEMALLOC_ATTR(pure)
   2587 je_sallocx(const void *ptr, int flags)
   2588 {
   2589 	size_t usize;
   2590 	tsdn_t *tsdn;
   2591 
   2592 	assert(malloc_initialized() || IS_INITIALIZER);
   2593 	malloc_thread_init();
   2594 
   2595 	tsdn = tsdn_fetch();
   2596 	witness_assert_lockless(tsdn);
   2597 
   2598 	if (config_ivsalloc)
   2599 		usize = ivsalloc(tsdn, ptr, config_prof);
   2600 	else
   2601 		usize = isalloc(tsdn, ptr, config_prof);
   2602 
   2603 	witness_assert_lockless(tsdn);
   2604 	return (usize);
   2605 }
   2606 
   2607 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
   2608 je_dallocx(void *ptr, int flags)
   2609 {
   2610 	tsd_t *tsd;
   2611 	tcache_t *tcache;
   2612 
   2613 	assert(ptr != NULL);
   2614 	assert(malloc_initialized() || IS_INITIALIZER);
   2615 
   2616 	tsd = tsd_fetch();
   2617 	witness_assert_lockless(tsd_tsdn(tsd));
   2618 	if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
   2619 		if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE)
   2620 			tcache = NULL;
   2621 		else
   2622 			tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
   2623 	} else
   2624 		tcache = tcache_get(tsd, false);
   2625 
   2626 	UTRACE(ptr, 0, 0);
   2627 	if (likely(!malloc_slow))
   2628 		ifree(tsd, ptr, tcache, false);
   2629 	else
   2630 		ifree(tsd, ptr, tcache, true);
   2631 	witness_assert_lockless(tsd_tsdn(tsd));
   2632 }
   2633 
   2634 JEMALLOC_ALWAYS_INLINE_C size_t
   2635 inallocx(tsdn_t *tsdn, size_t size, int flags)
   2636 {
   2637 	size_t usize;
   2638 
   2639 	witness_assert_lockless(tsdn);
   2640 
   2641 	if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0))
   2642 		usize = s2u(size);
   2643 	else
   2644 		usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags));
   2645 	witness_assert_lockless(tsdn);
   2646 	return (usize);
   2647 }
   2648 
   2649 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
   2650 je_sdallocx(void *ptr, size_t size, int flags)
   2651 {
   2652 	tsd_t *tsd;
   2653 	tcache_t *tcache;
   2654 	size_t usize;
   2655 
   2656 	assert(ptr != NULL);
   2657 	assert(malloc_initialized() || IS_INITIALIZER);
   2658 	tsd = tsd_fetch();
   2659 	usize = inallocx(tsd_tsdn(tsd), size, flags);
   2660 	assert(usize == isalloc(tsd_tsdn(tsd), ptr, config_prof));
   2661 
   2662 	witness_assert_lockless(tsd_tsdn(tsd));
   2663 	if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
   2664 		if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE)
   2665 			tcache = NULL;
   2666 		else
   2667 			tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
   2668 	} else
   2669 		tcache = tcache_get(tsd, false);
   2670 
   2671 	UTRACE(ptr, 0, 0);
   2672 	if (likely(!malloc_slow))
   2673 		isfree(tsd, ptr, usize, tcache, false);
   2674 	else
   2675 		isfree(tsd, ptr, usize, tcache, true);
   2676 	witness_assert_lockless(tsd_tsdn(tsd));
   2677 }
   2678 
   2679 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
   2680 JEMALLOC_ATTR(pure)
   2681 je_nallocx(size_t size, int flags)
   2682 {
   2683 	size_t usize;
   2684 	tsdn_t *tsdn;
   2685 
   2686 	assert(size != 0);
   2687 
   2688 	if (unlikely(malloc_init()))
   2689 		return (0);
   2690 
   2691 	tsdn = tsdn_fetch();
   2692 	witness_assert_lockless(tsdn);
   2693 
   2694 	usize = inallocx(tsdn, size, flags);
   2695 	if (unlikely(usize > HUGE_MAXCLASS))
   2696 		return (0);
   2697 
   2698 	witness_assert_lockless(tsdn);
   2699 	return (usize);
   2700 }
   2701 
   2702 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
   2703 je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
   2704     size_t newlen)
   2705 {
   2706 	int ret;
   2707 	tsd_t *tsd;
   2708 
   2709 	if (unlikely(malloc_init()))
   2710 		return (EAGAIN);
   2711 
   2712 	tsd = tsd_fetch();
   2713 	witness_assert_lockless(tsd_tsdn(tsd));
   2714 	ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen);
   2715 	witness_assert_lockless(tsd_tsdn(tsd));
   2716 	return (ret);
   2717 }
   2718 
   2719 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
   2720 je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp)
   2721 {
   2722 	int ret;
   2723 	tsdn_t *tsdn;
   2724 
   2725 	if (unlikely(malloc_init()))
   2726 		return (EAGAIN);
   2727 
   2728 	tsdn = tsdn_fetch();
   2729 	witness_assert_lockless(tsdn);
   2730 	ret = ctl_nametomib(tsdn, name, mibp, miblenp);
   2731 	witness_assert_lockless(tsdn);
   2732 	return (ret);
   2733 }
   2734 
   2735 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
   2736 je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
   2737   void *newp, size_t newlen)
   2738 {
   2739 	int ret;
   2740 	tsd_t *tsd;
   2741 
   2742 	if (unlikely(malloc_init()))
   2743 		return (EAGAIN);
   2744 
   2745 	tsd = tsd_fetch();
   2746 	witness_assert_lockless(tsd_tsdn(tsd));
   2747 	ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
   2748 	witness_assert_lockless(tsd_tsdn(tsd));
   2749 	return (ret);
   2750 }
   2751 
   2752 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
   2753 je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
   2754     const char *opts)
   2755 {
   2756 	tsdn_t *tsdn;
   2757 
   2758 	tsdn = tsdn_fetch();
   2759 	witness_assert_lockless(tsdn);
   2760 	stats_print(write_cb, cbopaque, opts);
   2761 	witness_assert_lockless(tsdn);
   2762 }
   2763 
   2764 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
   2765 je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
   2766 {
   2767 	size_t ret;
   2768 	tsdn_t *tsdn;
   2769 
   2770 	assert(malloc_initialized() || IS_INITIALIZER);
   2771 	malloc_thread_init();
   2772 
   2773 	tsdn = tsdn_fetch();
   2774 	witness_assert_lockless(tsdn);
   2775 
   2776 	if (config_ivsalloc)
   2777 		ret = ivsalloc(tsdn, ptr, config_prof);
   2778 	else
   2779 		ret = (ptr == NULL) ? 0 : isalloc(tsdn, ptr, config_prof);
   2780 
   2781 	witness_assert_lockless(tsdn);
   2782 	return (ret);
   2783 }
   2784 
   2785 /*
   2786  * End non-standard functions.
   2787  */
   2788 /******************************************************************************/
   2789 /*
   2790  * The following functions are used by threading libraries for protection of
   2791  * malloc during fork().
   2792  */
   2793 
   2794 /*
   2795  * If an application creates a thread before doing any allocation in the main
   2796  * thread, then calls fork(2) in the main thread followed by memory allocation
   2797  * in the child process, a race can occur that results in deadlock within the
   2798  * child: the main thread may have forked while the created thread had
   2799  * partially initialized the allocator.  Ordinarily jemalloc prevents
   2800  * fork/malloc races via the following functions it registers during
   2801  * initialization using pthread_atfork(), but of course that does no good if
   2802  * the allocator isn't fully initialized at fork time.  The following library
   2803  * constructor is a partial solution to this problem.  It may still be possible
   2804  * to trigger the deadlock described above, but doing so would involve forking
   2805  * via a library constructor that runs before jemalloc's runs.
   2806  */
   2807 #ifndef JEMALLOC_JET
   2808 JEMALLOC_ATTR(constructor)
   2809 static void
   2810 jemalloc_constructor(void)
   2811 {
   2812 
   2813 	malloc_init();
   2814 }
   2815 #endif
   2816 
   2817 #ifndef JEMALLOC_MUTEX_INIT_CB
   2818 void
   2819 jemalloc_prefork(void)
   2820 #else
   2821 JEMALLOC_EXPORT void
   2822 _malloc_prefork(void)
   2823 #endif
   2824 {
   2825 	tsd_t *tsd;
   2826 	unsigned i, j, narenas;
   2827 	arena_t *arena;
   2828 
   2829 #ifdef JEMALLOC_MUTEX_INIT_CB
   2830 	if (!malloc_initialized())
   2831 		return;
   2832 #endif
   2833 	assert(malloc_initialized());
   2834 
   2835 	tsd = tsd_fetch();
   2836 
   2837 	narenas = narenas_total_get();
   2838 
   2839 	witness_prefork(tsd);
   2840 	/* Acquire all mutexes in a safe order. */
   2841 	ctl_prefork(tsd_tsdn(tsd));
   2842 	tcache_prefork(tsd_tsdn(tsd));
   2843 	malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock);
   2844 	prof_prefork0(tsd_tsdn(tsd));
   2845 	for (i = 0; i < 3; i++) {
   2846 		for (j = 0; j < narenas; j++) {
   2847 			if ((arena = arena_get(tsd_tsdn(tsd), j, false)) !=
   2848 			    NULL) {
   2849 				switch (i) {
   2850 				case 0:
   2851 					arena_prefork0(tsd_tsdn(tsd), arena);
   2852 					break;
   2853 				case 1:
   2854 					arena_prefork1(tsd_tsdn(tsd), arena);
   2855 					break;
   2856 				case 2:
   2857 					arena_prefork2(tsd_tsdn(tsd), arena);
   2858 					break;
   2859 				default: not_reached();
   2860 				}
   2861 			}
   2862 		}
   2863 	}
   2864 	base_prefork(tsd_tsdn(tsd));
   2865 	for (i = 0; i < narenas; i++) {
   2866 		if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL)
   2867 			arena_prefork3(tsd_tsdn(tsd), arena);
   2868 	}
   2869 	prof_prefork1(tsd_tsdn(tsd));
   2870 }
   2871 
   2872 #ifndef JEMALLOC_MUTEX_INIT_CB
   2873 void
   2874 jemalloc_postfork_parent(void)
   2875 #else
   2876 JEMALLOC_EXPORT void
   2877 _malloc_postfork(void)
   2878 #endif
   2879 {
   2880 	tsd_t *tsd;
   2881 	unsigned i, narenas;
   2882 
   2883 #ifdef JEMALLOC_MUTEX_INIT_CB
   2884 	if (!malloc_initialized())
   2885 		return;
   2886 #endif
   2887 	assert(malloc_initialized());
   2888 
   2889 	tsd = tsd_fetch();
   2890 
   2891 	witness_postfork_parent(tsd);
   2892 	/* Release all mutexes, now that fork() has completed. */
   2893 	base_postfork_parent(tsd_tsdn(tsd));
   2894 	for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
   2895 		arena_t *arena;
   2896 
   2897 		if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL)
   2898 			arena_postfork_parent(tsd_tsdn(tsd), arena);
   2899 	}
   2900 	prof_postfork_parent(tsd_tsdn(tsd));
   2901 	malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock);
   2902 	tcache_postfork_parent(tsd_tsdn(tsd));
   2903 	ctl_postfork_parent(tsd_tsdn(tsd));
   2904 }
   2905 
   2906 void
   2907 jemalloc_postfork_child(void)
   2908 {
   2909 	tsd_t *tsd;
   2910 	unsigned i, narenas;
   2911 
   2912 	assert(malloc_initialized());
   2913 
   2914 	tsd = tsd_fetch();
   2915 
   2916 	witness_postfork_child(tsd);
   2917 	/* Release all mutexes, now that fork() has completed. */
   2918 	base_postfork_child(tsd_tsdn(tsd));
   2919 	for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
   2920 		arena_t *arena;
   2921 
   2922 		if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL)
   2923 			arena_postfork_child(tsd_tsdn(tsd), arena);
   2924 	}
   2925 	prof_postfork_child(tsd_tsdn(tsd));
   2926 	malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock);
   2927 	tcache_postfork_child(tsd_tsdn(tsd));
   2928 	ctl_postfork_child(tsd_tsdn(tsd));
   2929 }
   2930 
   2931 /******************************************************************************/
   2932 
   2933 #if defined(__BIONIC__)
   2934 /* ANDROID extension */
   2935 #include "android_je_iterate.c"
   2936 #include "android_je_mallinfo.c"
   2937 /* End ANDROID extension */
   2938 #endif
   2939