1 #define JEMALLOC_HUGE_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 4 /******************************************************************************/ 5 6 static extent_node_t * 7 huge_node_get(const void *ptr) 8 { 9 extent_node_t *node; 10 11 node = chunk_lookup(ptr, true); 12 assert(!extent_node_achunk_get(node)); 13 14 return (node); 15 } 16 17 static bool 18 huge_node_set(const void *ptr, extent_node_t *node) 19 { 20 21 assert(extent_node_addr_get(node) == ptr); 22 assert(!extent_node_achunk_get(node)); 23 return (chunk_register(ptr, node)); 24 } 25 26 static void 27 huge_node_unset(const void *ptr, const extent_node_t *node) 28 { 29 30 chunk_deregister(ptr, node); 31 } 32 33 void * 34 huge_malloc(tsd_t *tsd, arena_t *arena, size_t usize, bool zero, 35 tcache_t *tcache) 36 { 37 38 assert(usize == s2u(usize)); 39 40 return (huge_palloc(tsd, arena, usize, chunksize, zero, tcache)); 41 } 42 43 void * 44 huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, 45 bool zero, tcache_t *tcache) 46 { 47 void *ret; 48 size_t ausize; 49 extent_node_t *node; 50 bool is_zeroed; 51 52 /* Allocate one or more contiguous chunks for this request. */ 53 54 ausize = sa2u(usize, alignment); 55 if (unlikely(ausize == 0 || ausize > HUGE_MAXCLASS)) 56 return (NULL); 57 assert(ausize >= chunksize); 58 59 /* Allocate an extent node with which to track the chunk. */ 60 node = ipallocztm(tsd, CACHELINE_CEILING(sizeof(extent_node_t)), 61 CACHELINE, false, tcache, true, arena); 62 if (node == NULL) 63 return (NULL); 64 65 /* 66 * Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that 67 * it is possible to make correct junk/zero fill decisions below. 68 */ 69 is_zeroed = zero; 70 /* ANDROID change */ 71 #if !defined(__LP64__) 72 /* On 32 bit systems, using a per arena cache can exhaust 73 * virtual address space. Force all huge allocations to 74 * always take place in the first arena. 75 */ 76 extern arena_t *a0get(void); 77 arena = a0get(); 78 #else 79 arena = arena_choose(tsd, arena); 80 #endif 81 /* End ANDROID change */ 82 if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(arena, 83 usize, alignment, &is_zeroed)) == NULL) { 84 idalloctm(tsd, node, tcache, true, true); 85 return (NULL); 86 } 87 88 extent_node_init(node, arena, ret, usize, is_zeroed, true); 89 90 if (huge_node_set(ret, node)) { 91 arena_chunk_dalloc_huge(arena, ret, usize); 92 idalloctm(tsd, node, tcache, true, true); 93 return (NULL); 94 } 95 96 /* Insert node into huge. */ 97 malloc_mutex_lock(&arena->huge_mtx); 98 ql_elm_new(node, ql_link); 99 ql_tail_insert(&arena->huge, node, ql_link); 100 malloc_mutex_unlock(&arena->huge_mtx); 101 102 if (zero || (config_fill && unlikely(opt_zero))) { 103 if (!is_zeroed) 104 memset(ret, 0, usize); 105 } else if (config_fill && unlikely(opt_junk_alloc)) 106 memset(ret, 0xa5, usize); 107 108 arena_decay_tick(tsd, arena); 109 return (ret); 110 } 111 112 #ifdef JEMALLOC_JET 113 #undef huge_dalloc_junk 114 #define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl) 115 #endif 116 static void 117 huge_dalloc_junk(void *ptr, size_t usize) 118 { 119 120 if (config_fill && have_dss && unlikely(opt_junk_free)) { 121 /* 122 * Only bother junk filling if the chunk isn't about to be 123 * unmapped. 124 */ 125 if (!config_munmap || (have_dss && chunk_in_dss(ptr))) 126 memset(ptr, 0x5a, usize); 127 } 128 } 129 #ifdef JEMALLOC_JET 130 #undef huge_dalloc_junk 131 #define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk) 132 huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl); 133 #endif 134 135 static void 136 huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize_min, 137 size_t usize_max, bool zero) 138 { 139 size_t usize, usize_next; 140 extent_node_t *node; 141 arena_t *arena; 142 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; 143 bool pre_zeroed, post_zeroed; 144 145 /* Increase usize to incorporate extra. */ 146 for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1)) 147 <= oldsize; usize = usize_next) 148 ; /* Do nothing. */ 149 150 if (oldsize == usize) 151 return; 152 153 node = huge_node_get(ptr); 154 arena = extent_node_arena_get(node); 155 pre_zeroed = extent_node_zeroed_get(node); 156 157 /* Fill if necessary (shrinking). */ 158 if (oldsize > usize) { 159 size_t sdiff = oldsize - usize; 160 if (config_fill && unlikely(opt_junk_free)) { 161 memset((void *)((uintptr_t)ptr + usize), 0x5a, sdiff); 162 post_zeroed = false; 163 } else { 164 post_zeroed = !chunk_purge_wrapper(arena, &chunk_hooks, 165 ptr, CHUNK_CEILING(oldsize), usize, sdiff); 166 } 167 } else 168 post_zeroed = pre_zeroed; 169 170 malloc_mutex_lock(&arena->huge_mtx); 171 /* Update the size of the huge allocation. */ 172 assert(extent_node_size_get(node) != usize); 173 extent_node_size_set(node, usize); 174 /* Update zeroed. */ 175 extent_node_zeroed_set(node, post_zeroed); 176 malloc_mutex_unlock(&arena->huge_mtx); 177 178 arena_chunk_ralloc_huge_similar(arena, ptr, oldsize, usize); 179 180 /* Fill if necessary (growing). */ 181 if (oldsize < usize) { 182 if (zero || (config_fill && unlikely(opt_zero))) { 183 if (!pre_zeroed) { 184 memset((void *)((uintptr_t)ptr + oldsize), 0, 185 usize - oldsize); 186 } 187 } else if (config_fill && unlikely(opt_junk_alloc)) { 188 memset((void *)((uintptr_t)ptr + oldsize), 0xa5, usize - 189 oldsize); 190 } 191 } 192 } 193 194 static bool 195 huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) 196 { 197 extent_node_t *node; 198 arena_t *arena; 199 chunk_hooks_t chunk_hooks; 200 size_t cdiff; 201 bool pre_zeroed, post_zeroed; 202 203 node = huge_node_get(ptr); 204 arena = extent_node_arena_get(node); 205 pre_zeroed = extent_node_zeroed_get(node); 206 chunk_hooks = chunk_hooks_get(arena); 207 208 assert(oldsize > usize); 209 210 /* Split excess chunks. */ 211 cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); 212 if (cdiff != 0 && chunk_hooks.split(ptr, CHUNK_CEILING(oldsize), 213 CHUNK_CEILING(usize), cdiff, true, arena->ind)) 214 return (true); 215 216 if (oldsize > usize) { 217 size_t sdiff = oldsize - usize; 218 if (config_fill && unlikely(opt_junk_free)) { 219 huge_dalloc_junk((void *)((uintptr_t)ptr + usize), 220 sdiff); 221 post_zeroed = false; 222 } else { 223 post_zeroed = !chunk_purge_wrapper(arena, &chunk_hooks, 224 CHUNK_ADDR2BASE((uintptr_t)ptr + usize), 225 CHUNK_CEILING(oldsize), 226 CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff); 227 } 228 } else 229 post_zeroed = pre_zeroed; 230 231 malloc_mutex_lock(&arena->huge_mtx); 232 /* Update the size of the huge allocation. */ 233 extent_node_size_set(node, usize); 234 /* Update zeroed. */ 235 extent_node_zeroed_set(node, post_zeroed); 236 malloc_mutex_unlock(&arena->huge_mtx); 237 238 /* Zap the excess chunks. */ 239 arena_chunk_ralloc_huge_shrink(arena, ptr, oldsize, usize); 240 241 return (false); 242 } 243 244 static bool 245 huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t usize, bool zero) { 246 extent_node_t *node; 247 arena_t *arena; 248 bool is_zeroed_subchunk, is_zeroed_chunk; 249 250 node = huge_node_get(ptr); 251 arena = extent_node_arena_get(node); 252 malloc_mutex_lock(&arena->huge_mtx); 253 is_zeroed_subchunk = extent_node_zeroed_get(node); 254 malloc_mutex_unlock(&arena->huge_mtx); 255 256 /* 257 * Copy zero into is_zeroed_chunk and pass the copy to chunk_alloc(), so 258 * that it is possible to make correct junk/zero fill decisions below. 259 */ 260 is_zeroed_chunk = zero; 261 262 if (arena_chunk_ralloc_huge_expand(arena, ptr, oldsize, usize, 263 &is_zeroed_chunk)) 264 return (true); 265 266 malloc_mutex_lock(&arena->huge_mtx); 267 /* Update the size of the huge allocation. */ 268 extent_node_size_set(node, usize); 269 malloc_mutex_unlock(&arena->huge_mtx); 270 271 if (zero || (config_fill && unlikely(opt_zero))) { 272 if (!is_zeroed_subchunk) { 273 memset((void *)((uintptr_t)ptr + oldsize), 0, 274 CHUNK_CEILING(oldsize) - oldsize); 275 } 276 if (!is_zeroed_chunk) { 277 memset((void *)((uintptr_t)ptr + 278 CHUNK_CEILING(oldsize)), 0, usize - 279 CHUNK_CEILING(oldsize)); 280 } 281 } else if (config_fill && unlikely(opt_junk_alloc)) { 282 memset((void *)((uintptr_t)ptr + oldsize), 0xa5, usize - 283 oldsize); 284 } 285 286 return (false); 287 } 288 289 bool 290 huge_ralloc_no_move(tsd_t *tsd, void *ptr, size_t oldsize, size_t usize_min, 291 size_t usize_max, bool zero) 292 { 293 294 assert(s2u(oldsize) == oldsize); 295 /* The following should have been caught by callers. */ 296 assert(usize_min > 0 && usize_max <= HUGE_MAXCLASS); 297 298 /* Both allocations must be huge to avoid a move. */ 299 if (oldsize < chunksize || usize_max < chunksize) 300 return (true); 301 302 if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) { 303 /* Attempt to expand the allocation in-place. */ 304 if (!huge_ralloc_no_move_expand(ptr, oldsize, usize_max, 305 zero)) { 306 arena_decay_tick(tsd, huge_aalloc(ptr)); 307 return (false); 308 } 309 /* Try again, this time with usize_min. */ 310 if (usize_min < usize_max && CHUNK_CEILING(usize_min) > 311 CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(ptr, 312 oldsize, usize_min, zero)) { 313 arena_decay_tick(tsd, huge_aalloc(ptr)); 314 return (false); 315 } 316 } 317 318 /* 319 * Avoid moving the allocation if the existing chunk size accommodates 320 * the new size. 321 */ 322 if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize_min) 323 && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) { 324 huge_ralloc_no_move_similar(ptr, oldsize, usize_min, usize_max, 325 zero); 326 arena_decay_tick(tsd, huge_aalloc(ptr)); 327 return (false); 328 } 329 330 /* Attempt to shrink the allocation in-place. */ 331 if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max)) { 332 if (!huge_ralloc_no_move_shrink(ptr, oldsize, usize_max)) { 333 arena_decay_tick(tsd, huge_aalloc(ptr)); 334 return (false); 335 } 336 } 337 return (true); 338 } 339 340 static void * 341 huge_ralloc_move_helper(tsd_t *tsd, arena_t *arena, size_t usize, 342 size_t alignment, bool zero, tcache_t *tcache) 343 { 344 345 if (alignment <= chunksize) 346 return (huge_malloc(tsd, arena, usize, zero, tcache)); 347 return (huge_palloc(tsd, arena, usize, alignment, zero, tcache)); 348 } 349 350 void * 351 huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t usize, 352 size_t alignment, bool zero, tcache_t *tcache) 353 { 354 void *ret; 355 size_t copysize; 356 357 /* The following should have been caught by callers. */ 358 assert(usize > 0 && usize <= HUGE_MAXCLASS); 359 360 /* Try to avoid moving the allocation. */ 361 if (!huge_ralloc_no_move(tsd, ptr, oldsize, usize, usize, zero)) 362 return (ptr); 363 364 /* 365 * usize and oldsize are different enough that we need to use a 366 * different size class. In that case, fall back to allocating new 367 * space and copying. 368 */ 369 ret = huge_ralloc_move_helper(tsd, arena, usize, alignment, zero, 370 tcache); 371 if (ret == NULL) 372 return (NULL); 373 374 copysize = (usize < oldsize) ? usize : oldsize; 375 memcpy(ret, ptr, copysize); 376 isqalloc(tsd, ptr, oldsize, tcache); 377 return (ret); 378 } 379 380 void 381 huge_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache) 382 { 383 extent_node_t *node; 384 arena_t *arena; 385 386 node = huge_node_get(ptr); 387 arena = extent_node_arena_get(node); 388 huge_node_unset(ptr, node); 389 malloc_mutex_lock(&arena->huge_mtx); 390 ql_remove(&arena->huge, node, ql_link); 391 malloc_mutex_unlock(&arena->huge_mtx); 392 393 huge_dalloc_junk(extent_node_addr_get(node), 394 extent_node_size_get(node)); 395 arena_chunk_dalloc_huge(extent_node_arena_get(node), 396 extent_node_addr_get(node), extent_node_size_get(node)); 397 idalloctm(tsd, node, tcache, true, true); 398 399 arena_decay_tick(tsd, arena); 400 } 401 402 arena_t * 403 huge_aalloc(const void *ptr) 404 { 405 406 return (extent_node_arena_get(huge_node_get(ptr))); 407 } 408 409 size_t 410 huge_salloc(const void *ptr) 411 { 412 size_t size; 413 extent_node_t *node; 414 arena_t *arena; 415 416 node = huge_node_get(ptr); 417 arena = extent_node_arena_get(node); 418 malloc_mutex_lock(&arena->huge_mtx); 419 size = extent_node_size_get(node); 420 malloc_mutex_unlock(&arena->huge_mtx); 421 422 return (size); 423 } 424 425 prof_tctx_t * 426 huge_prof_tctx_get(const void *ptr) 427 { 428 prof_tctx_t *tctx; 429 extent_node_t *node; 430 arena_t *arena; 431 432 node = huge_node_get(ptr); 433 arena = extent_node_arena_get(node); 434 malloc_mutex_lock(&arena->huge_mtx); 435 tctx = extent_node_prof_tctx_get(node); 436 malloc_mutex_unlock(&arena->huge_mtx); 437 438 return (tctx); 439 } 440 441 void 442 huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) 443 { 444 extent_node_t *node; 445 arena_t *arena; 446 447 node = huge_node_get(ptr); 448 arena = extent_node_arena_get(node); 449 malloc_mutex_lock(&arena->huge_mtx); 450 extent_node_prof_tctx_set(node, tctx); 451 malloc_mutex_unlock(&arena->huge_mtx); 452 } 453 454 void 455 huge_prof_tctx_reset(const void *ptr) 456 { 457 458 huge_prof_tctx_set(ptr, (prof_tctx_t *)(uintptr_t)1U); 459 } 460
