Home | History | Annotate | Download | only in src
      1 #define	JEMALLOC_HUGE_C_
      2 #include "jemalloc/internal/jemalloc_internal.h"
      3 
      4 /******************************************************************************/
      5 
      6 static extent_node_t *
      7 huge_node_get(const void *ptr)
      8 {
      9 	extent_node_t *node;
     10 
     11 	node = chunk_lookup(ptr, true);
     12 	assert(!extent_node_achunk_get(node));
     13 
     14 	return (node);
     15 }
     16 
     17 static bool
     18 huge_node_set(tsdn_t *tsdn, const void *ptr, extent_node_t *node)
     19 {
     20 
     21 	assert(extent_node_addr_get(node) == ptr);
     22 	assert(!extent_node_achunk_get(node));
     23 	return (chunk_register(tsdn, ptr, node));
     24 }
     25 
     26 static void
     27 huge_node_reset(tsdn_t *tsdn, const void *ptr, extent_node_t *node)
     28 {
     29 	bool err;
     30 
     31 	err = huge_node_set(tsdn, ptr, node);
     32 	assert(!err);
     33 }
     34 
     35 static void
     36 huge_node_unset(const void *ptr, const extent_node_t *node)
     37 {
     38 
     39 	chunk_deregister(ptr, node);
     40 }
     41 
     42 void *
     43 huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero)
     44 {
     45 
     46 	assert(usize == s2u(usize));
     47 
     48 	return (huge_palloc(tsdn, arena, usize, chunksize, zero));
     49 }
     50 
     51 void *
     52 huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
     53     bool zero)
     54 {
     55 	void *ret;
     56 	size_t ausize;
     57 	arena_t *iarena;
     58 	extent_node_t *node;
     59 	size_t sn;
     60 	bool is_zeroed;
     61 
     62 	/* Allocate one or more contiguous chunks for this request. */
     63 
     64 	assert(!tsdn_null(tsdn) || arena != NULL);
     65 
     66 	ausize = sa2u(usize, alignment);
     67 	if (unlikely(ausize == 0 || ausize > HUGE_MAXCLASS))
     68 		return (NULL);
     69 	assert(ausize >= chunksize);
     70 
     71 	/* Allocate an extent node with which to track the chunk. */
     72 	iarena = (!tsdn_null(tsdn)) ? arena_ichoose(tsdn_tsd(tsdn), NULL) :
     73 	    a0get();
     74 	node = ipallocztm(tsdn, CACHELINE_CEILING(sizeof(extent_node_t)),
     75 	    CACHELINE, false, NULL, true, iarena);
     76 	if (node == NULL)
     77 		return (NULL);
     78 
     79 	/*
     80 	 * Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that
     81 	 * it is possible to make correct junk/zero fill decisions below.
     82 	 */
     83 	is_zeroed = zero;
     84 	/* ANDROID change */
     85 	if (likely(!tsdn_null(tsdn))) {
     86 #if defined(__ANDROID__) && !defined(__LP64__)
     87 		/* On 32 bit systems, using a per arena cache can exhaust
     88 		 * virtual address space. Force all huge allocations to
     89 		 * always take place in the first arena.
     90 		 */
     91 		extern arena_t *a0get(void);
     92 		arena = a0get();
     93 #else
     94 		arena = arena_choose(tsdn_tsd(tsdn), arena);
     95 #endif
     96 	}
     97 	/* End ANDROID change */
     98 	if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(tsdn,
     99 	    arena, usize, alignment, &sn, &is_zeroed)) == NULL) {
    100 		idalloctm(tsdn, node, NULL, true, true);
    101 		return (NULL);
    102 	}
    103 
    104 	extent_node_init(node, arena, ret, usize, sn, is_zeroed, true);
    105 
    106 	if (huge_node_set(tsdn, ret, node)) {
    107 		arena_chunk_dalloc_huge(tsdn, arena, ret, usize, sn);
    108 		idalloctm(tsdn, node, NULL, true, true);
    109 		return (NULL);
    110 	}
    111 
    112 	/* Insert node into huge. */
    113 	malloc_mutex_lock(tsdn, &arena->huge_mtx);
    114 	ql_elm_new(node, ql_link);
    115 	ql_tail_insert(&arena->huge, node, ql_link);
    116 	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
    117 
    118 	if (zero || (config_fill && unlikely(opt_zero))) {
    119 		if (!is_zeroed)
    120 			memset(ret, 0, usize);
    121 	} else if (config_fill && unlikely(opt_junk_alloc))
    122 		memset(ret, JEMALLOC_ALLOC_JUNK, usize);
    123 
    124 	arena_decay_tick(tsdn, arena);
    125 	return (ret);
    126 }
    127 
    128 #ifdef JEMALLOC_JET
    129 #undef huge_dalloc_junk
    130 #define	huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl)
    131 #endif
    132 static void
    133 huge_dalloc_junk(void *ptr, size_t usize)
    134 {
    135 
    136 	if (config_fill && have_dss && unlikely(opt_junk_free)) {
    137 		/*
    138 		 * Only bother junk filling if the chunk isn't about to be
    139 		 * unmapped.
    140 		 */
    141 		if (!config_munmap || (have_dss && chunk_in_dss(ptr)))
    142 			memset(ptr, JEMALLOC_FREE_JUNK, usize);
    143 	}
    144 }
    145 #ifdef JEMALLOC_JET
    146 #undef huge_dalloc_junk
    147 #define	huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk)
    148 huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
    149 #endif
    150 
    151 static void
    152 huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
    153     size_t usize_min, size_t usize_max, bool zero)
    154 {
    155 	size_t usize, usize_next;
    156 	extent_node_t *node;
    157 	arena_t *arena;
    158 	chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
    159 	bool pre_zeroed, post_zeroed;
    160 
    161 	/* Increase usize to incorporate extra. */
    162 	for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1))
    163 	    <= oldsize; usize = usize_next)
    164 		; /* Do nothing. */
    165 
    166 	if (oldsize == usize)
    167 		return;
    168 
    169 	node = huge_node_get(ptr);
    170 	arena = extent_node_arena_get(node);
    171 	pre_zeroed = extent_node_zeroed_get(node);
    172 
    173 	/* Fill if necessary (shrinking). */
    174 	if (oldsize > usize) {
    175 		size_t sdiff = oldsize - usize;
    176 		if (config_fill && unlikely(opt_junk_free)) {
    177 			memset((void *)((uintptr_t)ptr + usize),
    178 			    JEMALLOC_FREE_JUNK, sdiff);
    179 			post_zeroed = false;
    180 		} else {
    181 			post_zeroed = !chunk_purge_wrapper(tsdn, arena,
    182 			    &chunk_hooks, ptr, CHUNK_CEILING(oldsize), usize,
    183 			    sdiff);
    184 		}
    185 	} else
    186 		post_zeroed = pre_zeroed;
    187 
    188 	malloc_mutex_lock(tsdn, &arena->huge_mtx);
    189 	/* Update the size of the huge allocation. */
    190 	huge_node_unset(ptr, node);
    191 	assert(extent_node_size_get(node) != usize);
    192 	extent_node_size_set(node, usize);
    193 	huge_node_reset(tsdn, ptr, node);
    194 	/* Update zeroed. */
    195 	extent_node_zeroed_set(node, post_zeroed);
    196 	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
    197 
    198 	arena_chunk_ralloc_huge_similar(tsdn, arena, ptr, oldsize, usize);
    199 
    200 	/* Fill if necessary (growing). */
    201 	if (oldsize < usize) {
    202 		if (zero || (config_fill && unlikely(opt_zero))) {
    203 			if (!pre_zeroed) {
    204 				memset((void *)((uintptr_t)ptr + oldsize), 0,
    205 				    usize - oldsize);
    206 			}
    207 		} else if (config_fill && unlikely(opt_junk_alloc)) {
    208 			memset((void *)((uintptr_t)ptr + oldsize),
    209 			    JEMALLOC_ALLOC_JUNK, usize - oldsize);
    210 		}
    211 	}
    212 }
    213 
    214 static bool
    215 huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
    216     size_t usize)
    217 {
    218 	extent_node_t *node;
    219 	arena_t *arena;
    220 	chunk_hooks_t chunk_hooks;
    221 	size_t cdiff;
    222 	bool pre_zeroed, post_zeroed;
    223 
    224 	node = huge_node_get(ptr);
    225 	arena = extent_node_arena_get(node);
    226 	pre_zeroed = extent_node_zeroed_get(node);
    227 	chunk_hooks = chunk_hooks_get(tsdn, arena);
    228 
    229 	assert(oldsize > usize);
    230 
    231 	/* Split excess chunks. */
    232 	cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
    233 	if (cdiff != 0 && chunk_hooks.split(ptr, CHUNK_CEILING(oldsize),
    234 	    CHUNK_CEILING(usize), cdiff, true, arena->ind))
    235 		return (true);
    236 
    237 	if (oldsize > usize) {
    238 		size_t sdiff = oldsize - usize;
    239 		if (config_fill && unlikely(opt_junk_free)) {
    240 			huge_dalloc_junk((void *)((uintptr_t)ptr + usize),
    241 			    sdiff);
    242 			post_zeroed = false;
    243 		} else {
    244 			post_zeroed = !chunk_purge_wrapper(tsdn, arena,
    245 			    &chunk_hooks, CHUNK_ADDR2BASE((uintptr_t)ptr +
    246 			    usize), CHUNK_CEILING(oldsize),
    247 			    CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff);
    248 		}
    249 	} else
    250 		post_zeroed = pre_zeroed;
    251 
    252 	malloc_mutex_lock(tsdn, &arena->huge_mtx);
    253 	/* Update the size of the huge allocation. */
    254 	huge_node_unset(ptr, node);
    255 	extent_node_size_set(node, usize);
    256 	huge_node_reset(tsdn, ptr, node);
    257 	/* Update zeroed. */
    258 	extent_node_zeroed_set(node, post_zeroed);
    259 	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
    260 
    261 	/* Zap the excess chunks. */
    262 	arena_chunk_ralloc_huge_shrink(tsdn, arena, ptr, oldsize, usize,
    263 	    extent_node_sn_get(node));
    264 
    265 	return (false);
    266 }
    267 
    268 static bool
    269 huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
    270     size_t usize, bool zero) {
    271 	extent_node_t *node;
    272 	arena_t *arena;
    273 	bool is_zeroed_subchunk, is_zeroed_chunk;
    274 
    275 	node = huge_node_get(ptr);
    276 	arena = extent_node_arena_get(node);
    277 	malloc_mutex_lock(tsdn, &arena->huge_mtx);
    278 	is_zeroed_subchunk = extent_node_zeroed_get(node);
    279 	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
    280 
    281 	/*
    282 	 * Use is_zeroed_chunk to detect whether the trailing memory is zeroed,
    283 	 * update extent's zeroed field, and zero as necessary.
    284 	 */
    285 	is_zeroed_chunk = false;
    286 	if (arena_chunk_ralloc_huge_expand(tsdn, arena, ptr, oldsize, usize,
    287 	     &is_zeroed_chunk))
    288 		return (true);
    289 
    290 	malloc_mutex_lock(tsdn, &arena->huge_mtx);
    291 	huge_node_unset(ptr, node);
    292 	extent_node_size_set(node, usize);
    293 	extent_node_zeroed_set(node, extent_node_zeroed_get(node) &&
    294 	    is_zeroed_chunk);
    295 	huge_node_reset(tsdn, ptr, node);
    296 	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
    297 
    298 	if (zero || (config_fill && unlikely(opt_zero))) {
    299 		if (!is_zeroed_subchunk) {
    300 			memset((void *)((uintptr_t)ptr + oldsize), 0,
    301 			    CHUNK_CEILING(oldsize) - oldsize);
    302 		}
    303 		if (!is_zeroed_chunk) {
    304 			memset((void *)((uintptr_t)ptr +
    305 			    CHUNK_CEILING(oldsize)), 0, usize -
    306 			    CHUNK_CEILING(oldsize));
    307 		}
    308 	} else if (config_fill && unlikely(opt_junk_alloc)) {
    309 		memset((void *)((uintptr_t)ptr + oldsize), JEMALLOC_ALLOC_JUNK,
    310 		    usize - oldsize);
    311 	}
    312 
    313 	return (false);
    314 }
    315 
    316 bool
    317 huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
    318     size_t usize_max, bool zero)
    319 {
    320 
    321 	assert(s2u(oldsize) == oldsize);
    322 	/* The following should have been caught by callers. */
    323 	assert(usize_min > 0 && usize_max <= HUGE_MAXCLASS);
    324 
    325 	/* Both allocations must be huge to avoid a move. */
    326 	if (oldsize < chunksize || usize_max < chunksize)
    327 		return (true);
    328 
    329 	if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) {
    330 		/* Attempt to expand the allocation in-place. */
    331 		if (!huge_ralloc_no_move_expand(tsdn, ptr, oldsize, usize_max,
    332 		    zero)) {
    333 			arena_decay_tick(tsdn, huge_aalloc(ptr));
    334 			return (false);
    335 		}
    336 		/* Try again, this time with usize_min. */
    337 		if (usize_min < usize_max && CHUNK_CEILING(usize_min) >
    338 		    CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(tsdn,
    339 		    ptr, oldsize, usize_min, zero)) {
    340 			arena_decay_tick(tsdn, huge_aalloc(ptr));
    341 			return (false);
    342 		}
    343 	}
    344 
    345 	/*
    346 	 * Avoid moving the allocation if the existing chunk size accommodates
    347 	 * the new size.
    348 	 */
    349 	if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize_min)
    350 	    && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) {
    351 		huge_ralloc_no_move_similar(tsdn, ptr, oldsize, usize_min,
    352 		    usize_max, zero);
    353 		arena_decay_tick(tsdn, huge_aalloc(ptr));
    354 		return (false);
    355 	}
    356 
    357 	/* Attempt to shrink the allocation in-place. */
    358 	if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max)) {
    359 		if (!huge_ralloc_no_move_shrink(tsdn, ptr, oldsize,
    360 		    usize_max)) {
    361 			arena_decay_tick(tsdn, huge_aalloc(ptr));
    362 			return (false);
    363 		}
    364 	}
    365 	return (true);
    366 }
    367 
    368 static void *
    369 huge_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
    370     size_t alignment, bool zero)
    371 {
    372 
    373 	if (alignment <= chunksize)
    374 		return (huge_malloc(tsdn, arena, usize, zero));
    375 	return (huge_palloc(tsdn, arena, usize, alignment, zero));
    376 }
    377 
    378 void *
    379 huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize,
    380     size_t usize, size_t alignment, bool zero, tcache_t *tcache)
    381 {
    382 	void *ret;
    383 	size_t copysize;
    384 
    385 	/* The following should have been caught by callers. */
    386 	assert(usize > 0 && usize <= HUGE_MAXCLASS);
    387 
    388 	/* Try to avoid moving the allocation. */
    389 	if (!huge_ralloc_no_move(tsd_tsdn(tsd), ptr, oldsize, usize, usize,
    390 	    zero))
    391 		return (ptr);
    392 
    393 	/*
    394 	 * usize and oldsize are different enough that we need to use a
    395 	 * different size class.  In that case, fall back to allocating new
    396 	 * space and copying.
    397 	 */
    398 	ret = huge_ralloc_move_helper(tsd_tsdn(tsd), arena, usize, alignment,
    399 	    zero);
    400 	if (ret == NULL)
    401 		return (NULL);
    402 
    403 	copysize = (usize < oldsize) ? usize : oldsize;
    404 	memcpy(ret, ptr, copysize);
    405 	isqalloc(tsd, ptr, oldsize, tcache, true);
    406 	return (ret);
    407 }
    408 
    409 void
    410 huge_dalloc(tsdn_t *tsdn, void *ptr)
    411 {
    412 	extent_node_t *node;
    413 	arena_t *arena;
    414 
    415 	node = huge_node_get(ptr);
    416 	arena = extent_node_arena_get(node);
    417 	huge_node_unset(ptr, node);
    418 	malloc_mutex_lock(tsdn, &arena->huge_mtx);
    419 	ql_remove(&arena->huge, node, ql_link);
    420 	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
    421 
    422 	huge_dalloc_junk(extent_node_addr_get(node),
    423 	    extent_node_size_get(node));
    424 	arena_chunk_dalloc_huge(tsdn, extent_node_arena_get(node),
    425 	    extent_node_addr_get(node), extent_node_size_get(node),
    426 	    extent_node_sn_get(node));
    427 	idalloctm(tsdn, node, NULL, true, true);
    428 
    429 	arena_decay_tick(tsdn, arena);
    430 }
    431 
    432 arena_t *
    433 huge_aalloc(const void *ptr)
    434 {
    435 
    436 	return (extent_node_arena_get(huge_node_get(ptr)));
    437 }
    438 
    439 size_t
    440 huge_salloc(tsdn_t *tsdn, const void *ptr)
    441 {
    442 	size_t size;
    443 	extent_node_t *node;
    444 	arena_t *arena;
    445 
    446 	node = huge_node_get(ptr);
    447 	arena = extent_node_arena_get(node);
    448 	malloc_mutex_lock(tsdn, &arena->huge_mtx);
    449 	size = extent_node_size_get(node);
    450 	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
    451 
    452 	return (size);
    453 }
    454 
    455 prof_tctx_t *
    456 huge_prof_tctx_get(tsdn_t *tsdn, const void *ptr)
    457 {
    458 	prof_tctx_t *tctx;
    459 	extent_node_t *node;
    460 	arena_t *arena;
    461 
    462 	node = huge_node_get(ptr);
    463 	arena = extent_node_arena_get(node);
    464 	malloc_mutex_lock(tsdn, &arena->huge_mtx);
    465 	tctx = extent_node_prof_tctx_get(node);
    466 	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
    467 
    468 	return (tctx);
    469 }
    470 
    471 void
    472 huge_prof_tctx_set(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx)
    473 {
    474 	extent_node_t *node;
    475 	arena_t *arena;
    476 
    477 	node = huge_node_get(ptr);
    478 	arena = extent_node_arena_get(node);
    479 	malloc_mutex_lock(tsdn, &arena->huge_mtx);
    480 	extent_node_prof_tctx_set(node, tctx);
    481 	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
    482 }
    483 
    484 void
    485 huge_prof_tctx_reset(tsdn_t *tsdn, const void *ptr)
    486 {
    487 
    488 	huge_prof_tctx_set(tsdn, ptr, (prof_tctx_t *)(uintptr_t)1U);
    489 }
    490