1 #define JEMALLOC_HUGE_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 4 /******************************************************************************/ 5 6 static extent_node_t * 7 huge_node_get(const void *ptr) 8 { 9 extent_node_t *node; 10 11 node = chunk_lookup(ptr, true); 12 assert(!extent_node_achunk_get(node)); 13 14 return (node); 15 } 16 17 static bool 18 huge_node_set(tsdn_t *tsdn, const void *ptr, extent_node_t *node) 19 { 20 21 assert(extent_node_addr_get(node) == ptr); 22 assert(!extent_node_achunk_get(node)); 23 return (chunk_register(tsdn, ptr, node)); 24 } 25 26 static void 27 huge_node_reset(tsdn_t *tsdn, const void *ptr, extent_node_t *node) 28 { 29 bool err; 30 31 err = huge_node_set(tsdn, ptr, node); 32 assert(!err); 33 } 34 35 static void 36 huge_node_unset(const void *ptr, const extent_node_t *node) 37 { 38 39 chunk_deregister(ptr, node); 40 } 41 42 void * 43 huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero) 44 { 45 46 assert(usize == s2u(usize)); 47 48 return (huge_palloc(tsdn, arena, usize, chunksize, zero)); 49 } 50 51 void * 52 huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, 53 bool zero) 54 { 55 void *ret; 56 size_t ausize; 57 arena_t *iarena; 58 extent_node_t *node; 59 size_t sn; 60 bool is_zeroed; 61 62 /* Allocate one or more contiguous chunks for this request. */ 63 64 assert(!tsdn_null(tsdn) || arena != NULL); 65 66 ausize = sa2u(usize, alignment); 67 if (unlikely(ausize == 0 || ausize > HUGE_MAXCLASS)) 68 return (NULL); 69 assert(ausize >= chunksize); 70 71 /* Allocate an extent node with which to track the chunk. */ 72 iarena = (!tsdn_null(tsdn)) ? arena_ichoose(tsdn_tsd(tsdn), NULL) : 73 a0get(); 74 node = ipallocztm(tsdn, CACHELINE_CEILING(sizeof(extent_node_t)), 75 CACHELINE, false, NULL, true, iarena); 76 if (node == NULL) 77 return (NULL); 78 79 /* 80 * Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that 81 * it is possible to make correct junk/zero fill decisions below. 82 */ 83 is_zeroed = zero; 84 /* ANDROID change */ 85 if (likely(!tsdn_null(tsdn))) { 86 #if defined(__ANDROID__) && !defined(__LP64__) 87 /* On 32 bit systems, using a per arena cache can exhaust 88 * virtual address space. Force all huge allocations to 89 * always take place in the first arena. 90 */ 91 extern arena_t *a0get(void); 92 arena = a0get(); 93 #else 94 arena = arena_choose(tsdn_tsd(tsdn), arena); 95 #endif 96 } 97 /* End ANDROID change */ 98 if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(tsdn, 99 arena, usize, alignment, &sn, &is_zeroed)) == NULL) { 100 idalloctm(tsdn, node, NULL, true, true); 101 return (NULL); 102 } 103 104 extent_node_init(node, arena, ret, usize, sn, is_zeroed, true); 105 106 if (huge_node_set(tsdn, ret, node)) { 107 arena_chunk_dalloc_huge(tsdn, arena, ret, usize, sn); 108 idalloctm(tsdn, node, NULL, true, true); 109 return (NULL); 110 } 111 112 /* Insert node into huge. */ 113 malloc_mutex_lock(tsdn, &arena->huge_mtx); 114 ql_elm_new(node, ql_link); 115 ql_tail_insert(&arena->huge, node, ql_link); 116 malloc_mutex_unlock(tsdn, &arena->huge_mtx); 117 118 if (zero || (config_fill && unlikely(opt_zero))) { 119 if (!is_zeroed) 120 memset(ret, 0, usize); 121 } else if (config_fill && unlikely(opt_junk_alloc)) 122 memset(ret, JEMALLOC_ALLOC_JUNK, usize); 123 124 arena_decay_tick(tsdn, arena); 125 return (ret); 126 } 127 128 #ifdef JEMALLOC_JET 129 #undef huge_dalloc_junk 130 #define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl) 131 #endif 132 static void 133 huge_dalloc_junk(void *ptr, size_t usize) 134 { 135 136 if (config_fill && have_dss && unlikely(opt_junk_free)) { 137 /* 138 * Only bother junk filling if the chunk isn't about to be 139 * unmapped. 140 */ 141 if (!config_munmap || (have_dss && chunk_in_dss(ptr))) 142 memset(ptr, JEMALLOC_FREE_JUNK, usize); 143 } 144 } 145 #ifdef JEMALLOC_JET 146 #undef huge_dalloc_junk 147 #define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk) 148 huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl); 149 #endif 150 151 static void 152 huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize, 153 size_t usize_min, size_t usize_max, bool zero) 154 { 155 size_t usize, usize_next; 156 extent_node_t *node; 157 arena_t *arena; 158 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; 159 bool pre_zeroed, post_zeroed; 160 161 /* Increase usize to incorporate extra. */ 162 for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1)) 163 <= oldsize; usize = usize_next) 164 ; /* Do nothing. */ 165 166 if (oldsize == usize) 167 return; 168 169 node = huge_node_get(ptr); 170 arena = extent_node_arena_get(node); 171 pre_zeroed = extent_node_zeroed_get(node); 172 173 /* Fill if necessary (shrinking). */ 174 if (oldsize > usize) { 175 size_t sdiff = oldsize - usize; 176 if (config_fill && unlikely(opt_junk_free)) { 177 memset((void *)((uintptr_t)ptr + usize), 178 JEMALLOC_FREE_JUNK, sdiff); 179 post_zeroed = false; 180 } else { 181 post_zeroed = !chunk_purge_wrapper(tsdn, arena, 182 &chunk_hooks, ptr, CHUNK_CEILING(oldsize), usize, 183 sdiff); 184 } 185 } else 186 post_zeroed = pre_zeroed; 187 188 malloc_mutex_lock(tsdn, &arena->huge_mtx); 189 /* Update the size of the huge allocation. */ 190 huge_node_unset(ptr, node); 191 assert(extent_node_size_get(node) != usize); 192 extent_node_size_set(node, usize); 193 huge_node_reset(tsdn, ptr, node); 194 /* Update zeroed. */ 195 extent_node_zeroed_set(node, post_zeroed); 196 malloc_mutex_unlock(tsdn, &arena->huge_mtx); 197 198 arena_chunk_ralloc_huge_similar(tsdn, arena, ptr, oldsize, usize); 199 200 /* Fill if necessary (growing). */ 201 if (oldsize < usize) { 202 if (zero || (config_fill && unlikely(opt_zero))) { 203 if (!pre_zeroed) { 204 memset((void *)((uintptr_t)ptr + oldsize), 0, 205 usize - oldsize); 206 } 207 } else if (config_fill && unlikely(opt_junk_alloc)) { 208 memset((void *)((uintptr_t)ptr + oldsize), 209 JEMALLOC_ALLOC_JUNK, usize - oldsize); 210 } 211 } 212 } 213 214 static bool 215 huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize, 216 size_t usize) 217 { 218 extent_node_t *node; 219 arena_t *arena; 220 chunk_hooks_t chunk_hooks; 221 size_t cdiff; 222 bool pre_zeroed, post_zeroed; 223 224 node = huge_node_get(ptr); 225 arena = extent_node_arena_get(node); 226 pre_zeroed = extent_node_zeroed_get(node); 227 chunk_hooks = chunk_hooks_get(tsdn, arena); 228 229 assert(oldsize > usize); 230 231 /* Split excess chunks. */ 232 cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); 233 if (cdiff != 0 && chunk_hooks.split(ptr, CHUNK_CEILING(oldsize), 234 CHUNK_CEILING(usize), cdiff, true, arena->ind)) 235 return (true); 236 237 if (oldsize > usize) { 238 size_t sdiff = oldsize - usize; 239 if (config_fill && unlikely(opt_junk_free)) { 240 huge_dalloc_junk((void *)((uintptr_t)ptr + usize), 241 sdiff); 242 post_zeroed = false; 243 } else { 244 post_zeroed = !chunk_purge_wrapper(tsdn, arena, 245 &chunk_hooks, CHUNK_ADDR2BASE((uintptr_t)ptr + 246 usize), CHUNK_CEILING(oldsize), 247 CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff); 248 } 249 } else 250 post_zeroed = pre_zeroed; 251 252 malloc_mutex_lock(tsdn, &arena->huge_mtx); 253 /* Update the size of the huge allocation. */ 254 huge_node_unset(ptr, node); 255 extent_node_size_set(node, usize); 256 huge_node_reset(tsdn, ptr, node); 257 /* Update zeroed. */ 258 extent_node_zeroed_set(node, post_zeroed); 259 malloc_mutex_unlock(tsdn, &arena->huge_mtx); 260 261 /* Zap the excess chunks. */ 262 arena_chunk_ralloc_huge_shrink(tsdn, arena, ptr, oldsize, usize, 263 extent_node_sn_get(node)); 264 265 return (false); 266 } 267 268 static bool 269 huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize, 270 size_t usize, bool zero) { 271 extent_node_t *node; 272 arena_t *arena; 273 bool is_zeroed_subchunk, is_zeroed_chunk; 274 275 node = huge_node_get(ptr); 276 arena = extent_node_arena_get(node); 277 malloc_mutex_lock(tsdn, &arena->huge_mtx); 278 is_zeroed_subchunk = extent_node_zeroed_get(node); 279 malloc_mutex_unlock(tsdn, &arena->huge_mtx); 280 281 /* 282 * Use is_zeroed_chunk to detect whether the trailing memory is zeroed, 283 * update extent's zeroed field, and zero as necessary. 284 */ 285 is_zeroed_chunk = false; 286 if (arena_chunk_ralloc_huge_expand(tsdn, arena, ptr, oldsize, usize, 287 &is_zeroed_chunk)) 288 return (true); 289 290 malloc_mutex_lock(tsdn, &arena->huge_mtx); 291 huge_node_unset(ptr, node); 292 extent_node_size_set(node, usize); 293 extent_node_zeroed_set(node, extent_node_zeroed_get(node) && 294 is_zeroed_chunk); 295 huge_node_reset(tsdn, ptr, node); 296 malloc_mutex_unlock(tsdn, &arena->huge_mtx); 297 298 if (zero || (config_fill && unlikely(opt_zero))) { 299 if (!is_zeroed_subchunk) { 300 memset((void *)((uintptr_t)ptr + oldsize), 0, 301 CHUNK_CEILING(oldsize) - oldsize); 302 } 303 if (!is_zeroed_chunk) { 304 memset((void *)((uintptr_t)ptr + 305 CHUNK_CEILING(oldsize)), 0, usize - 306 CHUNK_CEILING(oldsize)); 307 } 308 } else if (config_fill && unlikely(opt_junk_alloc)) { 309 memset((void *)((uintptr_t)ptr + oldsize), JEMALLOC_ALLOC_JUNK, 310 usize - oldsize); 311 } 312 313 return (false); 314 } 315 316 bool 317 huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min, 318 size_t usize_max, bool zero) 319 { 320 321 assert(s2u(oldsize) == oldsize); 322 /* The following should have been caught by callers. */ 323 assert(usize_min > 0 && usize_max <= HUGE_MAXCLASS); 324 325 /* Both allocations must be huge to avoid a move. */ 326 if (oldsize < chunksize || usize_max < chunksize) 327 return (true); 328 329 if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) { 330 /* Attempt to expand the allocation in-place. */ 331 if (!huge_ralloc_no_move_expand(tsdn, ptr, oldsize, usize_max, 332 zero)) { 333 arena_decay_tick(tsdn, huge_aalloc(ptr)); 334 return (false); 335 } 336 /* Try again, this time with usize_min. */ 337 if (usize_min < usize_max && CHUNK_CEILING(usize_min) > 338 CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(tsdn, 339 ptr, oldsize, usize_min, zero)) { 340 arena_decay_tick(tsdn, huge_aalloc(ptr)); 341 return (false); 342 } 343 } 344 345 /* 346 * Avoid moving the allocation if the existing chunk size accommodates 347 * the new size. 348 */ 349 if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize_min) 350 && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) { 351 huge_ralloc_no_move_similar(tsdn, ptr, oldsize, usize_min, 352 usize_max, zero); 353 arena_decay_tick(tsdn, huge_aalloc(ptr)); 354 return (false); 355 } 356 357 /* Attempt to shrink the allocation in-place. */ 358 if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max)) { 359 if (!huge_ralloc_no_move_shrink(tsdn, ptr, oldsize, 360 usize_max)) { 361 arena_decay_tick(tsdn, huge_aalloc(ptr)); 362 return (false); 363 } 364 } 365 return (true); 366 } 367 368 static void * 369 huge_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, 370 size_t alignment, bool zero) 371 { 372 373 if (alignment <= chunksize) 374 return (huge_malloc(tsdn, arena, usize, zero)); 375 return (huge_palloc(tsdn, arena, usize, alignment, zero)); 376 } 377 378 void * 379 huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, 380 size_t usize, size_t alignment, bool zero, tcache_t *tcache) 381 { 382 void *ret; 383 size_t copysize; 384 385 /* The following should have been caught by callers. */ 386 assert(usize > 0 && usize <= HUGE_MAXCLASS); 387 388 /* Try to avoid moving the allocation. */ 389 if (!huge_ralloc_no_move(tsd_tsdn(tsd), ptr, oldsize, usize, usize, 390 zero)) 391 return (ptr); 392 393 /* 394 * usize and oldsize are different enough that we need to use a 395 * different size class. In that case, fall back to allocating new 396 * space and copying. 397 */ 398 ret = huge_ralloc_move_helper(tsd_tsdn(tsd), arena, usize, alignment, 399 zero); 400 if (ret == NULL) 401 return (NULL); 402 403 copysize = (usize < oldsize) ? usize : oldsize; 404 memcpy(ret, ptr, copysize); 405 isqalloc(tsd, ptr, oldsize, tcache, true); 406 return (ret); 407 } 408 409 void 410 huge_dalloc(tsdn_t *tsdn, void *ptr) 411 { 412 extent_node_t *node; 413 arena_t *arena; 414 415 node = huge_node_get(ptr); 416 arena = extent_node_arena_get(node); 417 huge_node_unset(ptr, node); 418 malloc_mutex_lock(tsdn, &arena->huge_mtx); 419 ql_remove(&arena->huge, node, ql_link); 420 malloc_mutex_unlock(tsdn, &arena->huge_mtx); 421 422 huge_dalloc_junk(extent_node_addr_get(node), 423 extent_node_size_get(node)); 424 arena_chunk_dalloc_huge(tsdn, extent_node_arena_get(node), 425 extent_node_addr_get(node), extent_node_size_get(node), 426 extent_node_sn_get(node)); 427 idalloctm(tsdn, node, NULL, true, true); 428 429 arena_decay_tick(tsdn, arena); 430 } 431 432 arena_t * 433 huge_aalloc(const void *ptr) 434 { 435 436 return (extent_node_arena_get(huge_node_get(ptr))); 437 } 438 439 size_t 440 huge_salloc(tsdn_t *tsdn, const void *ptr) 441 { 442 size_t size; 443 extent_node_t *node; 444 arena_t *arena; 445 446 node = huge_node_get(ptr); 447 arena = extent_node_arena_get(node); 448 malloc_mutex_lock(tsdn, &arena->huge_mtx); 449 size = extent_node_size_get(node); 450 malloc_mutex_unlock(tsdn, &arena->huge_mtx); 451 452 return (size); 453 } 454 455 prof_tctx_t * 456 huge_prof_tctx_get(tsdn_t *tsdn, const void *ptr) 457 { 458 prof_tctx_t *tctx; 459 extent_node_t *node; 460 arena_t *arena; 461 462 node = huge_node_get(ptr); 463 arena = extent_node_arena_get(node); 464 malloc_mutex_lock(tsdn, &arena->huge_mtx); 465 tctx = extent_node_prof_tctx_get(node); 466 malloc_mutex_unlock(tsdn, &arena->huge_mtx); 467 468 return (tctx); 469 } 470 471 void 472 huge_prof_tctx_set(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) 473 { 474 extent_node_t *node; 475 arena_t *arena; 476 477 node = huge_node_get(ptr); 478 arena = extent_node_arena_get(node); 479 malloc_mutex_lock(tsdn, &arena->huge_mtx); 480 extent_node_prof_tctx_set(node, tctx); 481 malloc_mutex_unlock(tsdn, &arena->huge_mtx); 482 } 483 484 void 485 huge_prof_tctx_reset(tsdn_t *tsdn, const void *ptr) 486 { 487 488 huge_prof_tctx_set(tsdn, ptr, (prof_tctx_t *)(uintptr_t)1U); 489 } 490
