1 /******************************************************************************/ 2 #ifdef JEMALLOC_H_TYPES 3 4 #define LARGE_MINCLASS (ZU(1) << LG_LARGE_MINCLASS) 5 6 /* Maximum number of regions in one run. */ 7 #define LG_RUN_MAXREGS (LG_PAGE - LG_TINY_MIN) 8 #define RUN_MAXREGS (1U << LG_RUN_MAXREGS) 9 10 /* 11 * Minimum redzone size. Redzones may be larger than this if necessary to 12 * preserve region alignment. 13 */ 14 #define REDZONE_MINSIZE 16 15 16 /* 17 * The minimum ratio of active:dirty pages per arena is computed as: 18 * 19 * (nactive >> lg_dirty_mult) >= ndirty 20 * 21 * So, supposing that lg_dirty_mult is 3, there can be no less than 8 times as 22 * many active pages as dirty pages. 23 */ 24 #define LG_DIRTY_MULT_DEFAULT 3 25 26 typedef enum { 27 purge_mode_ratio = 0, 28 purge_mode_decay = 1, 29 30 purge_mode_limit = 2 31 } purge_mode_t; 32 /* ANDROID change */ 33 /* Use the decay mode purge method. Do not set this value to zero, since 34 * forcing a purge immediately affects performance negatively. Using a 35 * small value provides a compromise between performance and extra PSS. 36 */ 37 #define PURGE_DEFAULT purge_mode_decay 38 /* Default decay time in seconds. */ 39 #define DECAY_TIME_DEFAULT 1 40 /* End ANDROID change */ 41 /* Number of event ticks between time checks. */ 42 #define DECAY_NTICKS_PER_UPDATE 1000 43 44 typedef struct arena_runs_dirty_link_s arena_runs_dirty_link_t; 45 typedef struct arena_run_s arena_run_t; 46 typedef struct arena_chunk_map_bits_s arena_chunk_map_bits_t; 47 typedef struct arena_chunk_map_misc_s arena_chunk_map_misc_t; 48 typedef struct arena_chunk_s arena_chunk_t; 49 typedef struct arena_bin_info_s arena_bin_info_t; 50 typedef struct arena_bin_s arena_bin_t; 51 typedef struct arena_s arena_t; 52 typedef struct arena_tdata_s arena_tdata_t; 53 54 #endif /* JEMALLOC_H_TYPES */ 55 /******************************************************************************/ 56 #ifdef JEMALLOC_H_STRUCTS 57 58 #ifdef JEMALLOC_ARENA_STRUCTS_A 59 struct arena_run_s { 60 /* Index of bin this run is associated with. */ 61 szind_t binind; 62 63 /* Number of free regions in run. */ 64 unsigned nfree; 65 66 /* Per region allocated/deallocated bitmap. */ 67 bitmap_t bitmap[BITMAP_GROUPS_MAX]; 68 }; 69 70 /* Each element of the chunk map corresponds to one page within the chunk. */ 71 struct arena_chunk_map_bits_s { 72 /* 73 * Run address (or size) and various flags are stored together. The bit 74 * layout looks like (assuming 32-bit system): 75 * 76 * ???????? ???????? ???nnnnn nnndumla 77 * 78 * ? : Unallocated: Run address for first/last pages, unset for internal 79 * pages. 80 * Small: Run page offset. 81 * Large: Run page count for first page, unset for trailing pages. 82 * n : binind for small size class, BININD_INVALID for large size class. 83 * d : dirty? 84 * u : unzeroed? 85 * m : decommitted? 86 * l : large? 87 * a : allocated? 88 * 89 * Following are example bit patterns for the three types of runs. 90 * 91 * p : run page offset 92 * s : run size 93 * n : binind for size class; large objects set these to BININD_INVALID 94 * x : don't care 95 * - : 0 96 * + : 1 97 * [DUMLA] : bit set 98 * [dumla] : bit unset 99 * 100 * Unallocated (clean): 101 * ssssssss ssssssss sss+++++ +++dum-a 102 * xxxxxxxx xxxxxxxx xxxxxxxx xxx-Uxxx 103 * ssssssss ssssssss sss+++++ +++dUm-a 104 * 105 * Unallocated (dirty): 106 * ssssssss ssssssss sss+++++ +++D-m-a 107 * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 108 * ssssssss ssssssss sss+++++ +++D-m-a 109 * 110 * Small: 111 * pppppppp pppppppp pppnnnnn nnnd---A 112 * pppppppp pppppppp pppnnnnn nnn----A 113 * pppppppp pppppppp pppnnnnn nnnd---A 114 * 115 * Large: 116 * ssssssss ssssssss sss+++++ +++D--LA 117 * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 118 * -------- -------- ---+++++ +++D--LA 119 * 120 * Large (sampled, size <= LARGE_MINCLASS): 121 * ssssssss ssssssss sssnnnnn nnnD--LA 122 * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 123 * -------- -------- ---+++++ +++D--LA 124 * 125 * Large (not sampled, size == LARGE_MINCLASS): 126 * ssssssss ssssssss sss+++++ +++D--LA 127 * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 128 * -------- -------- ---+++++ +++D--LA 129 */ 130 size_t bits; 131 #define CHUNK_MAP_ALLOCATED ((size_t)0x01U) 132 #define CHUNK_MAP_LARGE ((size_t)0x02U) 133 #define CHUNK_MAP_STATE_MASK ((size_t)0x3U) 134 135 #define CHUNK_MAP_DECOMMITTED ((size_t)0x04U) 136 #define CHUNK_MAP_UNZEROED ((size_t)0x08U) 137 #define CHUNK_MAP_DIRTY ((size_t)0x10U) 138 #define CHUNK_MAP_FLAGS_MASK ((size_t)0x1cU) 139 140 #define CHUNK_MAP_BININD_SHIFT 5 141 #define BININD_INVALID ((size_t)0xffU) 142 #define CHUNK_MAP_BININD_MASK (BININD_INVALID << CHUNK_MAP_BININD_SHIFT) 143 #define CHUNK_MAP_BININD_INVALID CHUNK_MAP_BININD_MASK 144 145 #define CHUNK_MAP_RUNIND_SHIFT (CHUNK_MAP_BININD_SHIFT + 8) 146 #define CHUNK_MAP_SIZE_SHIFT (CHUNK_MAP_RUNIND_SHIFT - LG_PAGE) 147 #define CHUNK_MAP_SIZE_MASK \ 148 (~(CHUNK_MAP_BININD_MASK | CHUNK_MAP_FLAGS_MASK | CHUNK_MAP_STATE_MASK)) 149 }; 150 151 struct arena_runs_dirty_link_s { 152 qr(arena_runs_dirty_link_t) rd_link; 153 }; 154 155 /* 156 * Each arena_chunk_map_misc_t corresponds to one page within the chunk, just 157 * like arena_chunk_map_bits_t. Two separate arrays are stored within each 158 * chunk header in order to improve cache locality. 159 */ 160 struct arena_chunk_map_misc_s { 161 /* 162 * Linkage for run trees. There are two disjoint uses: 163 * 164 * 1) arena_t's runs_avail tree. 165 * 2) arena_run_t conceptually uses this linkage for in-use non-full 166 * runs, rather than directly embedding linkage. 167 */ 168 rb_node(arena_chunk_map_misc_t) rb_link; 169 170 union { 171 /* Linkage for list of dirty runs. */ 172 arena_runs_dirty_link_t rd; 173 174 /* Profile counters, used for large object runs. */ 175 union { 176 void *prof_tctx_pun; 177 prof_tctx_t *prof_tctx; 178 }; 179 180 /* Small region run metadata. */ 181 arena_run_t run; 182 }; 183 }; 184 typedef rb_tree(arena_chunk_map_misc_t) arena_run_tree_t; 185 #endif /* JEMALLOC_ARENA_STRUCTS_A */ 186 187 #ifdef JEMALLOC_ARENA_STRUCTS_B 188 /* Arena chunk header. */ 189 struct arena_chunk_s { 190 /* 191 * A pointer to the arena that owns the chunk is stored within the node. 192 * This field as a whole is used by chunks_rtree to support both 193 * ivsalloc() and core-based debugging. 194 */ 195 extent_node_t node; 196 197 /* 198 * Map of pages within chunk that keeps track of free/large/small. The 199 * first map_bias entries are omitted, since the chunk header does not 200 * need to be tracked in the map. This omission saves a header page 201 * for common chunk sizes (e.g. 4 MiB). 202 */ 203 arena_chunk_map_bits_t map_bits[1]; /* Dynamically sized. */ 204 }; 205 206 /* 207 * Read-only information associated with each element of arena_t's bins array 208 * is stored separately, partly to reduce memory usage (only one copy, rather 209 * than one per arena), but mainly to avoid false cacheline sharing. 210 * 211 * Each run has the following layout: 212 * 213 * /--------------------\ 214 * | pad? | 215 * |--------------------| 216 * | redzone | 217 * reg0_offset | region 0 | 218 * | redzone | 219 * |--------------------| \ 220 * | redzone | | 221 * | region 1 | > reg_interval 222 * | redzone | / 223 * |--------------------| 224 * | ... | 225 * | ... | 226 * | ... | 227 * |--------------------| 228 * | redzone | 229 * | region nregs-1 | 230 * | redzone | 231 * |--------------------| 232 * | alignment pad? | 233 * \--------------------/ 234 * 235 * reg_interval has at least the same minimum alignment as reg_size; this 236 * preserves the alignment constraint that sa2u() depends on. Alignment pad is 237 * either 0 or redzone_size; it is present only if needed to align reg0_offset. 238 */ 239 struct arena_bin_info_s { 240 /* Size of regions in a run for this bin's size class. */ 241 size_t reg_size; 242 243 /* Redzone size. */ 244 size_t redzone_size; 245 246 /* Interval between regions (reg_size + (redzone_size << 1)). */ 247 size_t reg_interval; 248 249 /* Total size of a run for this bin's size class. */ 250 size_t run_size; 251 252 /* Total number of regions in a run for this bin's size class. */ 253 uint32_t nregs; 254 255 /* 256 * Metadata used to manipulate bitmaps for runs associated with this 257 * bin. 258 */ 259 bitmap_info_t bitmap_info; 260 261 /* Offset of first region in a run for this bin's size class. */ 262 uint32_t reg0_offset; 263 }; 264 265 struct arena_bin_s { 266 /* 267 * All operations on runcur, runs, and stats require that lock be 268 * locked. Run allocation/deallocation are protected by the arena lock, 269 * which may be acquired while holding one or more bin locks, but not 270 * vise versa. 271 */ 272 malloc_mutex_t lock; 273 274 /* 275 * Current run being used to service allocations of this bin's size 276 * class. 277 */ 278 arena_run_t *runcur; 279 280 /* 281 * Tree of non-full runs. This tree is used when looking for an 282 * existing run when runcur is no longer usable. We choose the 283 * non-full run that is lowest in memory; this policy tends to keep 284 * objects packed well, and it can also help reduce the number of 285 * almost-empty chunks. 286 */ 287 arena_run_tree_t runs; 288 289 /* Bin statistics. */ 290 malloc_bin_stats_t stats; 291 }; 292 293 struct arena_s { 294 /* This arena's index within the arenas array. */ 295 unsigned ind; 296 297 /* 298 * Number of threads currently assigned to this arena. This field is 299 * synchronized via atomic operations. 300 */ 301 unsigned nthreads; 302 303 /* 304 * There are three classes of arena operations from a locking 305 * perspective: 306 * 1) Thread assignment (modifies nthreads) is synchronized via atomics. 307 * 2) Bin-related operations are protected by bin locks. 308 * 3) Chunk- and run-related operations are protected by this mutex. 309 */ 310 malloc_mutex_t lock; 311 312 arena_stats_t stats; 313 /* 314 * List of tcaches for extant threads associated with this arena. 315 * Stats from these are merged incrementally, and at exit if 316 * opt_stats_print is enabled. 317 */ 318 ql_head(tcache_t) tcache_ql; 319 320 uint64_t prof_accumbytes; 321 322 /* 323 * PRNG state for cache index randomization of large allocation base 324 * pointers. 325 */ 326 uint64_t offset_state; 327 328 dss_prec_t dss_prec; 329 330 /* 331 * In order to avoid rapid chunk allocation/deallocation when an arena 332 * oscillates right on the cusp of needing a new chunk, cache the most 333 * recently freed chunk. The spare is left in the arena's chunk trees 334 * until it is deleted. 335 * 336 * There is one spare chunk per arena, rather than one spare total, in 337 * order to avoid interactions between multiple threads that could make 338 * a single spare inadequate. 339 */ 340 arena_chunk_t *spare; 341 342 /* Minimum ratio (log base 2) of nactive:ndirty. */ 343 ssize_t lg_dirty_mult; 344 345 /* True if a thread is currently executing arena_purge_to_limit(). */ 346 bool purging; 347 348 /* Number of pages in active runs and huge regions. */ 349 size_t nactive; 350 351 /* 352 * Current count of pages within unused runs that are potentially 353 * dirty, and for which madvise(... MADV_DONTNEED) has not been called. 354 * By tracking this, we can institute a limit on how much dirty unused 355 * memory is mapped for each arena. 356 */ 357 size_t ndirty; 358 359 /* 360 * Unused dirty memory this arena manages. Dirty memory is conceptually 361 * tracked as an arbitrarily interleaved LRU of dirty runs and cached 362 * chunks, but the list linkage is actually semi-duplicated in order to 363 * avoid extra arena_chunk_map_misc_t space overhead. 364 * 365 * LRU-----------------------------------------------------------MRU 366 * 367 * /-- arena ---\ 368 * | | 369 * | | 370 * |------------| /- chunk -\ 371 * ...->|chunks_cache|<--------------------------->| /----\ |<--... 372 * |------------| | |node| | 373 * | | | | | | 374 * | | /- run -\ /- run -\ | | | | 375 * | | | | | | | | | | 376 * | | | | | | | | | | 377 * |------------| |-------| |-------| | |----| | 378 * ...->|runs_dirty |<-->|rd |<-->|rd |<---->|rd |<----... 379 * |------------| |-------| |-------| | |----| | 380 * | | | | | | | | | | 381 * | | | | | | | \----/ | 382 * | | \-------/ \-------/ | | 383 * | | | | 384 * | | | | 385 * \------------/ \---------/ 386 */ 387 arena_runs_dirty_link_t runs_dirty; 388 extent_node_t chunks_cache; 389 390 /* 391 * Approximate time in seconds from the creation of a set of unused 392 * dirty pages until an equivalent set of unused dirty pages is purged 393 * and/or reused. 394 */ 395 ssize_t decay_time; 396 /* decay_time / SMOOTHSTEP_NSTEPS. */ 397 nstime_t decay_interval; 398 /* 399 * Time at which the current decay interval logically started. We do 400 * not actually advance to a new epoch until sometime after it starts 401 * because of scheduling and computation delays, and it is even possible 402 * to completely skip epochs. In all cases, during epoch advancement we 403 * merge all relevant activity into the most recently recorded epoch. 404 */ 405 nstime_t decay_epoch; 406 /* decay_deadline randomness generator. */ 407 uint64_t decay_jitter_state; 408 /* 409 * Deadline for current epoch. This is the sum of decay_interval and 410 * per epoch jitter which is a uniform random variable in 411 * [0..decay_interval). Epochs always advance by precise multiples of 412 * decay_interval, but we randomize the deadline to reduce the 413 * likelihood of arenas purging in lockstep. 414 */ 415 nstime_t decay_deadline; 416 /* 417 * Number of dirty pages at beginning of current epoch. During epoch 418 * advancement we use the delta between decay_ndirty and ndirty to 419 * determine how many dirty pages, if any, were generated, and record 420 * the result in decay_backlog. 421 */ 422 size_t decay_ndirty; 423 /* 424 * Memoized result of arena_decay_backlog_npages_limit() corresponding 425 * to the current contents of decay_backlog, i.e. the limit on how many 426 * pages are allowed to exist for the decay epochs. 427 */ 428 size_t decay_backlog_npages_limit; 429 /* 430 * Trailing log of how many unused dirty pages were generated during 431 * each of the past SMOOTHSTEP_NSTEPS decay epochs, where the last 432 * element is the most recent epoch. Corresponding epoch times are 433 * relative to decay_epoch. 434 */ 435 size_t decay_backlog[SMOOTHSTEP_NSTEPS]; 436 437 /* Extant huge allocations. */ 438 ql_head(extent_node_t) huge; 439 /* Synchronizes all huge allocation/update/deallocation. */ 440 malloc_mutex_t huge_mtx; 441 442 /* 443 * Trees of chunks that were previously allocated (trees differ only in 444 * node ordering). These are used when allocating chunks, in an attempt 445 * to re-use address space. Depending on function, different tree 446 * orderings are needed, which is why there are two trees with the same 447 * contents. 448 */ 449 extent_tree_t chunks_szad_cached; 450 extent_tree_t chunks_ad_cached; 451 extent_tree_t chunks_szad_retained; 452 extent_tree_t chunks_ad_retained; 453 454 malloc_mutex_t chunks_mtx; 455 /* Cache of nodes that were allocated via base_alloc(). */ 456 ql_head(extent_node_t) node_cache; 457 malloc_mutex_t node_cache_mtx; 458 459 /* User-configurable chunk hook functions. */ 460 chunk_hooks_t chunk_hooks; 461 462 /* bins is used to store trees of free regions. */ 463 arena_bin_t bins[NBINS]; 464 465 /* 466 * Quantized address-ordered trees of this arena's available runs. The 467 * trees are used for first-best-fit run allocation. 468 */ 469 arena_run_tree_t runs_avail[1]; /* Dynamically sized. */ 470 }; 471 472 /* Used in conjunction with tsd for fast arena-related context lookup. */ 473 struct arena_tdata_s { 474 ticker_t decay_ticker; 475 }; 476 #endif /* JEMALLOC_ARENA_STRUCTS_B */ 477 478 #endif /* JEMALLOC_H_STRUCTS */ 479 /******************************************************************************/ 480 #ifdef JEMALLOC_H_EXTERNS 481 482 static const size_t large_pad = 483 #ifdef JEMALLOC_CACHE_OBLIVIOUS 484 PAGE 485 #else 486 0 487 #endif 488 ; 489 490 extern purge_mode_t opt_purge; 491 extern const char *purge_mode_names[]; 492 extern ssize_t opt_lg_dirty_mult; 493 extern ssize_t opt_decay_time; 494 495 extern arena_bin_info_t arena_bin_info[NBINS]; 496 497 extern size_t map_bias; /* Number of arena chunk header pages. */ 498 extern size_t map_misc_offset; 499 extern size_t arena_maxrun; /* Max run size for arenas. */ 500 extern size_t large_maxclass; /* Max large size class. */ 501 extern size_t run_quantize_max; /* Max run_quantize_*() input. */ 502 extern unsigned nlclasses; /* Number of large size classes. */ 503 extern unsigned nhclasses; /* Number of huge size classes. */ 504 505 #ifdef JEMALLOC_JET 506 typedef size_t (run_quantize_t)(size_t); 507 extern run_quantize_t *run_quantize_floor; 508 extern run_quantize_t *run_quantize_ceil; 509 #endif 510 void arena_chunk_cache_maybe_insert(arena_t *arena, extent_node_t *node, 511 bool cache); 512 void arena_chunk_cache_maybe_remove(arena_t *arena, extent_node_t *node, 513 bool cache); 514 extent_node_t *arena_node_alloc(arena_t *arena); 515 void arena_node_dalloc(arena_t *arena, extent_node_t *node); 516 void *arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment, 517 bool *zero); 518 void arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize); 519 void arena_chunk_ralloc_huge_similar(arena_t *arena, void *chunk, 520 size_t oldsize, size_t usize); 521 void arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk, 522 size_t oldsize, size_t usize); 523 bool arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, 524 size_t oldsize, size_t usize, bool *zero); 525 ssize_t arena_lg_dirty_mult_get(arena_t *arena); 526 bool arena_lg_dirty_mult_set(arena_t *arena, ssize_t lg_dirty_mult); 527 ssize_t arena_decay_time_get(arena_t *arena); 528 bool arena_decay_time_set(arena_t *arena, ssize_t decay_time); 529 void arena_maybe_purge(arena_t *arena); 530 void arena_purge(arena_t *arena, bool all); 531 void arena_tcache_fill_small(tsd_t *tsd, arena_t *arena, tcache_bin_t *tbin, 532 szind_t binind, uint64_t prof_accumbytes); 533 void arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, 534 bool zero); 535 #ifdef JEMALLOC_JET 536 typedef void (arena_redzone_corruption_t)(void *, size_t, bool, size_t, 537 uint8_t); 538 extern arena_redzone_corruption_t *arena_redzone_corruption; 539 typedef void (arena_dalloc_junk_small_t)(void *, arena_bin_info_t *); 540 extern arena_dalloc_junk_small_t *arena_dalloc_junk_small; 541 #else 542 void arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info); 543 #endif 544 void arena_quarantine_junk_small(void *ptr, size_t usize); 545 void *arena_malloc_large(tsd_t *tsd, arena_t *arena, szind_t ind, bool zero); 546 void *arena_malloc_hard(tsd_t *tsd, arena_t *arena, size_t size, szind_t ind, 547 bool zero, tcache_t *tcache); 548 void *arena_palloc(tsd_t *tsd, arena_t *arena, size_t usize, 549 size_t alignment, bool zero, tcache_t *tcache); 550 void arena_prof_promoted(const void *ptr, size_t size); 551 void arena_dalloc_bin_junked_locked(arena_t *arena, arena_chunk_t *chunk, 552 void *ptr, arena_chunk_map_bits_t *bitselm); 553 void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 554 size_t pageind, arena_chunk_map_bits_t *bitselm); 555 void arena_dalloc_small(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk, 556 void *ptr, size_t pageind); 557 #ifdef JEMALLOC_JET 558 typedef void (arena_dalloc_junk_large_t)(void *, size_t); 559 extern arena_dalloc_junk_large_t *arena_dalloc_junk_large; 560 #else 561 void arena_dalloc_junk_large(void *ptr, size_t usize); 562 #endif 563 void arena_dalloc_large_junked_locked(arena_t *arena, arena_chunk_t *chunk, 564 void *ptr); 565 void arena_dalloc_large(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk, 566 void *ptr); 567 #ifdef JEMALLOC_JET 568 typedef void (arena_ralloc_junk_large_t)(void *, size_t, size_t); 569 extern arena_ralloc_junk_large_t *arena_ralloc_junk_large; 570 #endif 571 bool arena_ralloc_no_move(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, 572 size_t extra, bool zero); 573 void *arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, 574 size_t size, size_t alignment, bool zero, tcache_t *tcache); 575 dss_prec_t arena_dss_prec_get(arena_t *arena); 576 bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); 577 ssize_t arena_lg_dirty_mult_default_get(void); 578 bool arena_lg_dirty_mult_default_set(ssize_t lg_dirty_mult); 579 ssize_t arena_decay_time_default_get(void); 580 bool arena_decay_time_default_set(ssize_t decay_time); 581 void arena_basic_stats_merge(arena_t *arena, unsigned *nthreads, 582 const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time, 583 size_t *nactive, size_t *ndirty); 584 void arena_stats_merge(arena_t *arena, unsigned *nthreads, const char **dss, 585 ssize_t *lg_dirty_mult, ssize_t *decay_time, size_t *nactive, 586 size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, 587 malloc_large_stats_t *lstats, malloc_huge_stats_t *hstats); 588 unsigned arena_nthreads_get(arena_t *arena); 589 void arena_nthreads_inc(arena_t *arena); 590 void arena_nthreads_dec(arena_t *arena); 591 arena_t *arena_new(unsigned ind); 592 bool arena_boot(void); 593 void arena_prefork(arena_t *arena); 594 void arena_postfork_parent(arena_t *arena); 595 void arena_postfork_child(arena_t *arena); 596 597 #endif /* JEMALLOC_H_EXTERNS */ 598 /******************************************************************************/ 599 #ifdef JEMALLOC_H_INLINES 600 601 #ifndef JEMALLOC_ENABLE_INLINE 602 arena_chunk_map_bits_t *arena_bitselm_get(arena_chunk_t *chunk, 603 size_t pageind); 604 arena_chunk_map_misc_t *arena_miscelm_get(arena_chunk_t *chunk, 605 size_t pageind); 606 size_t arena_miscelm_to_pageind(const arena_chunk_map_misc_t *miscelm); 607 void *arena_miscelm_to_rpages(arena_chunk_map_misc_t *miscelm); 608 arena_chunk_map_misc_t *arena_rd_to_miscelm(arena_runs_dirty_link_t *rd); 609 arena_chunk_map_misc_t *arena_run_to_miscelm(arena_run_t *run); 610 size_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind); 611 size_t arena_mapbitsp_read(size_t *mapbitsp); 612 size_t arena_mapbits_get(arena_chunk_t *chunk, size_t pageind); 613 size_t arena_mapbits_size_decode(size_t mapbits); 614 size_t arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, 615 size_t pageind); 616 size_t arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind); 617 size_t arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind); 618 szind_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind); 619 size_t arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind); 620 size_t arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind); 621 size_t arena_mapbits_decommitted_get(arena_chunk_t *chunk, size_t pageind); 622 size_t arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind); 623 size_t arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind); 624 void arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits); 625 size_t arena_mapbits_size_encode(size_t size); 626 void arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, 627 size_t size, size_t flags); 628 void arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 629 size_t size); 630 void arena_mapbits_internal_set(arena_chunk_t *chunk, size_t pageind, 631 size_t flags); 632 void arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, 633 size_t size, size_t flags); 634 void arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 635 szind_t binind); 636 void arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, 637 size_t runind, szind_t binind, size_t flags); 638 void arena_metadata_allocated_add(arena_t *arena, size_t size); 639 void arena_metadata_allocated_sub(arena_t *arena, size_t size); 640 size_t arena_metadata_allocated_get(arena_t *arena); 641 bool arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes); 642 bool arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes); 643 bool arena_prof_accum(arena_t *arena, uint64_t accumbytes); 644 szind_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits); 645 szind_t arena_bin_index(arena_t *arena, arena_bin_t *bin); 646 size_t arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, 647 const void *ptr); 648 prof_tctx_t *arena_prof_tctx_get(const void *ptr); 649 void arena_prof_tctx_set(const void *ptr, size_t usize, prof_tctx_t *tctx); 650 void arena_prof_tctx_reset(const void *ptr, size_t usize, 651 const void *old_ptr, prof_tctx_t *old_tctx); 652 void arena_decay_ticks(tsd_t *tsd, arena_t *arena, unsigned nticks); 653 void arena_decay_tick(tsd_t *tsd, arena_t *arena); 654 void *arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, szind_t ind, 655 bool zero, tcache_t *tcache, bool slow_path); 656 arena_t *arena_aalloc(const void *ptr); 657 size_t arena_salloc(const void *ptr, bool demote); 658 void arena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path); 659 void arena_sdalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache); 660 #endif 661 662 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) 663 # ifdef JEMALLOC_ARENA_INLINE_A 664 JEMALLOC_ALWAYS_INLINE arena_chunk_map_bits_t * 665 arena_bitselm_get(arena_chunk_t *chunk, size_t pageind) 666 { 667 668 assert(pageind >= map_bias); 669 assert(pageind < chunk_npages); 670 671 return (&chunk->map_bits[pageind-map_bias]); 672 } 673 674 JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * 675 arena_miscelm_get(arena_chunk_t *chunk, size_t pageind) 676 { 677 678 assert(pageind >= map_bias); 679 assert(pageind < chunk_npages); 680 681 return ((arena_chunk_map_misc_t *)((uintptr_t)chunk + 682 (uintptr_t)map_misc_offset) + pageind-map_bias); 683 } 684 685 JEMALLOC_ALWAYS_INLINE size_t 686 arena_miscelm_to_pageind(const arena_chunk_map_misc_t *miscelm) 687 { 688 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); 689 size_t pageind = ((uintptr_t)miscelm - ((uintptr_t)chunk + 690 map_misc_offset)) / sizeof(arena_chunk_map_misc_t) + map_bias; 691 692 assert(pageind >= map_bias); 693 assert(pageind < chunk_npages); 694 695 return (pageind); 696 } 697 698 JEMALLOC_ALWAYS_INLINE void * 699 arena_miscelm_to_rpages(arena_chunk_map_misc_t *miscelm) 700 { 701 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); 702 size_t pageind = arena_miscelm_to_pageind(miscelm); 703 704 return ((void *)((uintptr_t)chunk + (pageind << LG_PAGE))); 705 } 706 707 JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * 708 arena_rd_to_miscelm(arena_runs_dirty_link_t *rd) 709 { 710 arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t 711 *)((uintptr_t)rd - offsetof(arena_chunk_map_misc_t, rd)); 712 713 assert(arena_miscelm_to_pageind(miscelm) >= map_bias); 714 assert(arena_miscelm_to_pageind(miscelm) < chunk_npages); 715 716 return (miscelm); 717 } 718 719 JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * 720 arena_run_to_miscelm(arena_run_t *run) 721 { 722 arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t 723 *)((uintptr_t)run - offsetof(arena_chunk_map_misc_t, run)); 724 725 assert(arena_miscelm_to_pageind(miscelm) >= map_bias); 726 assert(arena_miscelm_to_pageind(miscelm) < chunk_npages); 727 728 return (miscelm); 729 } 730 731 JEMALLOC_ALWAYS_INLINE size_t * 732 arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind) 733 { 734 735 return (&arena_bitselm_get(chunk, pageind)->bits); 736 } 737 738 JEMALLOC_ALWAYS_INLINE size_t 739 arena_mapbitsp_read(size_t *mapbitsp) 740 { 741 742 return (*mapbitsp); 743 } 744 745 JEMALLOC_ALWAYS_INLINE size_t 746 arena_mapbits_get(arena_chunk_t *chunk, size_t pageind) 747 { 748 749 return (arena_mapbitsp_read(arena_mapbitsp_get(chunk, pageind))); 750 } 751 752 JEMALLOC_ALWAYS_INLINE size_t 753 arena_mapbits_size_decode(size_t mapbits) 754 { 755 size_t size; 756 757 #if CHUNK_MAP_SIZE_SHIFT > 0 758 size = (mapbits & CHUNK_MAP_SIZE_MASK) >> CHUNK_MAP_SIZE_SHIFT; 759 #elif CHUNK_MAP_SIZE_SHIFT == 0 760 size = mapbits & CHUNK_MAP_SIZE_MASK; 761 #else 762 size = (mapbits & CHUNK_MAP_SIZE_MASK) << -CHUNK_MAP_SIZE_SHIFT; 763 #endif 764 765 return (size); 766 } 767 768 JEMALLOC_ALWAYS_INLINE size_t 769 arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind) 770 { 771 size_t mapbits; 772 773 mapbits = arena_mapbits_get(chunk, pageind); 774 assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 775 return (arena_mapbits_size_decode(mapbits)); 776 } 777 778 JEMALLOC_ALWAYS_INLINE size_t 779 arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind) 780 { 781 size_t mapbits; 782 783 mapbits = arena_mapbits_get(chunk, pageind); 784 assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 785 (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)); 786 return (arena_mapbits_size_decode(mapbits)); 787 } 788 789 JEMALLOC_ALWAYS_INLINE size_t 790 arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind) 791 { 792 size_t mapbits; 793 794 mapbits = arena_mapbits_get(chunk, pageind); 795 assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 796 CHUNK_MAP_ALLOCATED); 797 return (mapbits >> CHUNK_MAP_RUNIND_SHIFT); 798 } 799 800 JEMALLOC_ALWAYS_INLINE szind_t 801 arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind) 802 { 803 size_t mapbits; 804 szind_t binind; 805 806 mapbits = arena_mapbits_get(chunk, pageind); 807 binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 808 assert(binind < NBINS || binind == BININD_INVALID); 809 return (binind); 810 } 811 812 JEMALLOC_ALWAYS_INLINE size_t 813 arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind) 814 { 815 size_t mapbits; 816 817 mapbits = arena_mapbits_get(chunk, pageind); 818 assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits & 819 (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 820 return (mapbits & CHUNK_MAP_DIRTY); 821 } 822 823 JEMALLOC_ALWAYS_INLINE size_t 824 arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind) 825 { 826 size_t mapbits; 827 828 mapbits = arena_mapbits_get(chunk, pageind); 829 assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits & 830 (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 831 return (mapbits & CHUNK_MAP_UNZEROED); 832 } 833 834 JEMALLOC_ALWAYS_INLINE size_t 835 arena_mapbits_decommitted_get(arena_chunk_t *chunk, size_t pageind) 836 { 837 size_t mapbits; 838 839 mapbits = arena_mapbits_get(chunk, pageind); 840 assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits & 841 (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 842 return (mapbits & CHUNK_MAP_DECOMMITTED); 843 } 844 845 JEMALLOC_ALWAYS_INLINE size_t 846 arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind) 847 { 848 size_t mapbits; 849 850 mapbits = arena_mapbits_get(chunk, pageind); 851 return (mapbits & CHUNK_MAP_LARGE); 852 } 853 854 JEMALLOC_ALWAYS_INLINE size_t 855 arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind) 856 { 857 size_t mapbits; 858 859 mapbits = arena_mapbits_get(chunk, pageind); 860 return (mapbits & CHUNK_MAP_ALLOCATED); 861 } 862 863 JEMALLOC_ALWAYS_INLINE void 864 arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits) 865 { 866 867 *mapbitsp = mapbits; 868 } 869 870 JEMALLOC_ALWAYS_INLINE size_t 871 arena_mapbits_size_encode(size_t size) 872 { 873 size_t mapbits; 874 875 #if CHUNK_MAP_SIZE_SHIFT > 0 876 mapbits = size << CHUNK_MAP_SIZE_SHIFT; 877 #elif CHUNK_MAP_SIZE_SHIFT == 0 878 mapbits = size; 879 #else 880 mapbits = size >> -CHUNK_MAP_SIZE_SHIFT; 881 #endif 882 883 assert((mapbits & ~CHUNK_MAP_SIZE_MASK) == 0); 884 return (mapbits); 885 } 886 887 JEMALLOC_ALWAYS_INLINE void 888 arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size, 889 size_t flags) 890 { 891 size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 892 893 assert((size & PAGE_MASK) == 0); 894 assert((flags & CHUNK_MAP_FLAGS_MASK) == flags); 895 assert((flags & CHUNK_MAP_DECOMMITTED) == 0 || (flags & 896 (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 897 arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) | 898 CHUNK_MAP_BININD_INVALID | flags); 899 } 900 901 JEMALLOC_ALWAYS_INLINE void 902 arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 903 size_t size) 904 { 905 size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 906 size_t mapbits = arena_mapbitsp_read(mapbitsp); 907 908 assert((size & PAGE_MASK) == 0); 909 assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 910 arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) | 911 (mapbits & ~CHUNK_MAP_SIZE_MASK)); 912 } 913 914 JEMALLOC_ALWAYS_INLINE void 915 arena_mapbits_internal_set(arena_chunk_t *chunk, size_t pageind, size_t flags) 916 { 917 size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 918 919 assert((flags & CHUNK_MAP_UNZEROED) == flags); 920 arena_mapbitsp_write(mapbitsp, flags); 921 } 922 923 JEMALLOC_ALWAYS_INLINE void 924 arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size, 925 size_t flags) 926 { 927 size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 928 929 assert((size & PAGE_MASK) == 0); 930 assert((flags & CHUNK_MAP_FLAGS_MASK) == flags); 931 assert((flags & CHUNK_MAP_DECOMMITTED) == 0 || (flags & 932 (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 933 arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) | 934 CHUNK_MAP_BININD_INVALID | flags | CHUNK_MAP_LARGE | 935 CHUNK_MAP_ALLOCATED); 936 } 937 938 JEMALLOC_ALWAYS_INLINE void 939 arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 940 szind_t binind) 941 { 942 size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 943 size_t mapbits = arena_mapbitsp_read(mapbitsp); 944 945 assert(binind <= BININD_INVALID); 946 assert(arena_mapbits_large_size_get(chunk, pageind) == LARGE_MINCLASS + 947 large_pad); 948 arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_BININD_MASK) | 949 (binind << CHUNK_MAP_BININD_SHIFT)); 950 } 951 952 JEMALLOC_ALWAYS_INLINE void 953 arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind, 954 szind_t binind, size_t flags) 955 { 956 size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 957 958 assert(binind < BININD_INVALID); 959 assert(pageind - runind >= map_bias); 960 assert((flags & CHUNK_MAP_UNZEROED) == flags); 961 arena_mapbitsp_write(mapbitsp, (runind << CHUNK_MAP_RUNIND_SHIFT) | 962 (binind << CHUNK_MAP_BININD_SHIFT) | flags | CHUNK_MAP_ALLOCATED); 963 } 964 965 JEMALLOC_INLINE void 966 arena_metadata_allocated_add(arena_t *arena, size_t size) 967 { 968 969 atomic_add_z(&arena->stats.metadata_allocated, size); 970 } 971 972 JEMALLOC_INLINE void 973 arena_metadata_allocated_sub(arena_t *arena, size_t size) 974 { 975 976 atomic_sub_z(&arena->stats.metadata_allocated, size); 977 } 978 979 JEMALLOC_INLINE size_t 980 arena_metadata_allocated_get(arena_t *arena) 981 { 982 983 return (atomic_read_z(&arena->stats.metadata_allocated)); 984 } 985 986 JEMALLOC_INLINE bool 987 arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes) 988 { 989 990 cassert(config_prof); 991 assert(prof_interval != 0); 992 993 arena->prof_accumbytes += accumbytes; 994 if (arena->prof_accumbytes >= prof_interval) { 995 arena->prof_accumbytes -= prof_interval; 996 return (true); 997 } 998 return (false); 999 } 1000 1001 JEMALLOC_INLINE bool 1002 arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes) 1003 { 1004 1005 cassert(config_prof); 1006 1007 if (likely(prof_interval == 0)) 1008 return (false); 1009 return (arena_prof_accum_impl(arena, accumbytes)); 1010 } 1011 1012 JEMALLOC_INLINE bool 1013 arena_prof_accum(arena_t *arena, uint64_t accumbytes) 1014 { 1015 1016 cassert(config_prof); 1017 1018 if (likely(prof_interval == 0)) 1019 return (false); 1020 1021 { 1022 bool ret; 1023 1024 malloc_mutex_lock(&arena->lock); 1025 ret = arena_prof_accum_impl(arena, accumbytes); 1026 malloc_mutex_unlock(&arena->lock); 1027 return (ret); 1028 } 1029 } 1030 1031 JEMALLOC_ALWAYS_INLINE szind_t 1032 arena_ptr_small_binind_get(const void *ptr, size_t mapbits) 1033 { 1034 szind_t binind; 1035 1036 binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 1037 1038 if (config_debug) { 1039 arena_chunk_t *chunk; 1040 arena_t *arena; 1041 size_t pageind; 1042 size_t actual_mapbits; 1043 size_t rpages_ind; 1044 arena_run_t *run; 1045 arena_bin_t *bin; 1046 szind_t run_binind, actual_binind; 1047 arena_bin_info_t *bin_info; 1048 arena_chunk_map_misc_t *miscelm; 1049 void *rpages; 1050 1051 assert(binind != BININD_INVALID); 1052 assert(binind < NBINS); 1053 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1054 arena = extent_node_arena_get(&chunk->node); 1055 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1056 actual_mapbits = arena_mapbits_get(chunk, pageind); 1057 assert(mapbits == actual_mapbits); 1058 assert(arena_mapbits_large_get(chunk, pageind) == 0); 1059 assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 1060 rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, 1061 pageind); 1062 miscelm = arena_miscelm_get(chunk, rpages_ind); 1063 run = &miscelm->run; 1064 run_binind = run->binind; 1065 bin = &arena->bins[run_binind]; 1066 actual_binind = (szind_t)(bin - arena->bins); 1067 assert(run_binind == actual_binind); 1068 bin_info = &arena_bin_info[actual_binind]; 1069 rpages = arena_miscelm_to_rpages(miscelm); 1070 assert(((uintptr_t)ptr - ((uintptr_t)rpages + 1071 (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval 1072 == 0); 1073 } 1074 1075 return (binind); 1076 } 1077 # endif /* JEMALLOC_ARENA_INLINE_A */ 1078 1079 # ifdef JEMALLOC_ARENA_INLINE_B 1080 JEMALLOC_INLINE szind_t 1081 arena_bin_index(arena_t *arena, arena_bin_t *bin) 1082 { 1083 szind_t binind = (szind_t)(bin - arena->bins); 1084 assert(binind < NBINS); 1085 return (binind); 1086 } 1087 1088 JEMALLOC_INLINE size_t 1089 arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) 1090 { 1091 size_t diff, interval, shift, regind; 1092 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 1093 void *rpages = arena_miscelm_to_rpages(miscelm); 1094 1095 /* 1096 * Freeing a pointer lower than region zero can cause assertion 1097 * failure. 1098 */ 1099 assert((uintptr_t)ptr >= (uintptr_t)rpages + 1100 (uintptr_t)bin_info->reg0_offset); 1101 1102 /* 1103 * Avoid doing division with a variable divisor if possible. Using 1104 * actual division here can reduce allocator throughput by over 20%! 1105 */ 1106 diff = (size_t)((uintptr_t)ptr - (uintptr_t)rpages - 1107 bin_info->reg0_offset); 1108 1109 /* Rescale (factor powers of 2 out of the numerator and denominator). */ 1110 interval = bin_info->reg_interval; 1111 shift = ffs_zu(interval) - 1; 1112 diff >>= shift; 1113 interval >>= shift; 1114 1115 if (interval == 1) { 1116 /* The divisor was a power of 2. */ 1117 regind = diff; 1118 } else { 1119 /* 1120 * To divide by a number D that is not a power of two we 1121 * multiply by (2^21 / D) and then right shift by 21 positions. 1122 * 1123 * X / D 1124 * 1125 * becomes 1126 * 1127 * (X * interval_invs[D - 3]) >> SIZE_INV_SHIFT 1128 * 1129 * We can omit the first three elements, because we never 1130 * divide by 0, and 1 and 2 are both powers of two, which are 1131 * handled above. 1132 */ 1133 #define SIZE_INV_SHIFT ((sizeof(size_t) << 3) - LG_RUN_MAXREGS) 1134 #define SIZE_INV(s) (((ZU(1) << SIZE_INV_SHIFT) / (s)) + 1) 1135 static const size_t interval_invs[] = { 1136 SIZE_INV(3), 1137 SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), 1138 SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), 1139 SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15), 1140 SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19), 1141 SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23), 1142 SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27), 1143 SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) 1144 }; 1145 1146 if (likely(interval <= ((sizeof(interval_invs) / sizeof(size_t)) 1147 + 2))) { 1148 regind = (diff * interval_invs[interval - 3]) >> 1149 SIZE_INV_SHIFT; 1150 } else 1151 regind = diff / interval; 1152 #undef SIZE_INV 1153 #undef SIZE_INV_SHIFT 1154 } 1155 assert(diff == regind * interval); 1156 assert(regind < bin_info->nregs); 1157 1158 return (regind); 1159 } 1160 1161 JEMALLOC_INLINE prof_tctx_t * 1162 arena_prof_tctx_get(const void *ptr) 1163 { 1164 prof_tctx_t *ret; 1165 arena_chunk_t *chunk; 1166 1167 cassert(config_prof); 1168 assert(ptr != NULL); 1169 1170 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1171 if (likely(chunk != ptr)) { 1172 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1173 size_t mapbits = arena_mapbits_get(chunk, pageind); 1174 assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 1175 if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) 1176 ret = (prof_tctx_t *)(uintptr_t)1U; 1177 else { 1178 arena_chunk_map_misc_t *elm = arena_miscelm_get(chunk, 1179 pageind); 1180 ret = atomic_read_p(&elm->prof_tctx_pun); 1181 } 1182 } else 1183 ret = huge_prof_tctx_get(ptr); 1184 1185 return (ret); 1186 } 1187 1188 JEMALLOC_INLINE void 1189 arena_prof_tctx_set(const void *ptr, size_t usize, prof_tctx_t *tctx) 1190 { 1191 arena_chunk_t *chunk; 1192 1193 cassert(config_prof); 1194 assert(ptr != NULL); 1195 1196 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1197 if (likely(chunk != ptr)) { 1198 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1199 1200 assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 1201 1202 if (unlikely(usize > SMALL_MAXCLASS || (uintptr_t)tctx > 1203 (uintptr_t)1U)) { 1204 arena_chunk_map_misc_t *elm; 1205 1206 assert(arena_mapbits_large_get(chunk, pageind) != 0); 1207 1208 elm = arena_miscelm_get(chunk, pageind); 1209 atomic_write_p(&elm->prof_tctx_pun, tctx); 1210 } else { 1211 /* 1212 * tctx must always be initialized for large runs. 1213 * Assert that the surrounding conditional logic is 1214 * equivalent to checking whether ptr refers to a large 1215 * run. 1216 */ 1217 assert(arena_mapbits_large_get(chunk, pageind) == 0); 1218 } 1219 } else 1220 huge_prof_tctx_set(ptr, tctx); 1221 } 1222 1223 JEMALLOC_INLINE void 1224 arena_prof_tctx_reset(const void *ptr, size_t usize, const void *old_ptr, 1225 prof_tctx_t *old_tctx) 1226 { 1227 1228 cassert(config_prof); 1229 assert(ptr != NULL); 1230 1231 if (unlikely(usize > SMALL_MAXCLASS || (ptr == old_ptr && 1232 (uintptr_t)old_tctx > (uintptr_t)1U))) { 1233 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1234 if (likely(chunk != ptr)) { 1235 size_t pageind; 1236 arena_chunk_map_misc_t *elm; 1237 1238 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> 1239 LG_PAGE; 1240 assert(arena_mapbits_allocated_get(chunk, pageind) != 1241 0); 1242 assert(arena_mapbits_large_get(chunk, pageind) != 0); 1243 1244 elm = arena_miscelm_get(chunk, pageind); 1245 atomic_write_p(&elm->prof_tctx_pun, 1246 (prof_tctx_t *)(uintptr_t)1U); 1247 } else 1248 huge_prof_tctx_reset(ptr); 1249 } 1250 } 1251 1252 JEMALLOC_ALWAYS_INLINE void 1253 arena_decay_ticks(tsd_t *tsd, arena_t *arena, unsigned nticks) 1254 { 1255 ticker_t *decay_ticker; 1256 1257 if (unlikely(tsd == NULL)) 1258 return; 1259 decay_ticker = decay_ticker_get(tsd, arena->ind); 1260 if (unlikely(decay_ticker == NULL)) 1261 return; 1262 if (unlikely(ticker_ticks(decay_ticker, nticks))) 1263 arena_purge(arena, false); 1264 } 1265 1266 JEMALLOC_ALWAYS_INLINE void 1267 arena_decay_tick(tsd_t *tsd, arena_t *arena) 1268 { 1269 1270 arena_decay_ticks(tsd, arena, 1); 1271 } 1272 1273 JEMALLOC_ALWAYS_INLINE void * 1274 arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, szind_t ind, bool zero, 1275 tcache_t *tcache, bool slow_path) 1276 { 1277 1278 assert(size != 0); 1279 1280 if (likely(tcache != NULL)) { 1281 if (likely(size <= SMALL_MAXCLASS)) { 1282 return (tcache_alloc_small(tsd, arena, tcache, size, 1283 ind, zero, slow_path)); 1284 } 1285 if (likely(size <= tcache_maxclass)) { 1286 return (tcache_alloc_large(tsd, arena, tcache, size, 1287 ind, zero, slow_path)); 1288 } 1289 /* (size > tcache_maxclass) case falls through. */ 1290 assert(size > tcache_maxclass); 1291 } 1292 1293 return (arena_malloc_hard(tsd, arena, size, ind, zero, tcache)); 1294 } 1295 1296 JEMALLOC_ALWAYS_INLINE arena_t * 1297 arena_aalloc(const void *ptr) 1298 { 1299 arena_chunk_t *chunk; 1300 1301 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1302 if (likely(chunk != ptr)) 1303 return (extent_node_arena_get(&chunk->node)); 1304 else 1305 return (huge_aalloc(ptr)); 1306 } 1307 1308 /* Return the size of the allocation pointed to by ptr. */ 1309 JEMALLOC_ALWAYS_INLINE size_t 1310 arena_salloc(const void *ptr, bool demote) 1311 { 1312 size_t ret; 1313 arena_chunk_t *chunk; 1314 size_t pageind; 1315 szind_t binind; 1316 1317 assert(ptr != NULL); 1318 1319 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1320 if (likely(chunk != ptr)) { 1321 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1322 assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 1323 binind = arena_mapbits_binind_get(chunk, pageind); 1324 if (unlikely(binind == BININD_INVALID || (config_prof && !demote 1325 && arena_mapbits_large_get(chunk, pageind) != 0))) { 1326 /* 1327 * Large allocation. In the common case (demote), and 1328 * as this is an inline function, most callers will only 1329 * end up looking at binind to determine that ptr is a 1330 * small allocation. 1331 */ 1332 assert(config_cache_oblivious || ((uintptr_t)ptr & 1333 PAGE_MASK) == 0); 1334 ret = arena_mapbits_large_size_get(chunk, pageind) - 1335 large_pad; 1336 assert(ret != 0); 1337 assert(pageind + ((ret+large_pad)>>LG_PAGE) <= 1338 chunk_npages); 1339 assert(arena_mapbits_dirty_get(chunk, pageind) == 1340 arena_mapbits_dirty_get(chunk, 1341 pageind+((ret+large_pad)>>LG_PAGE)-1)); 1342 } else { 1343 /* 1344 * Small allocation (possibly promoted to a large 1345 * object). 1346 */ 1347 assert(arena_mapbits_large_get(chunk, pageind) != 0 || 1348 arena_ptr_small_binind_get(ptr, 1349 arena_mapbits_get(chunk, pageind)) == binind); 1350 ret = index2size(binind); 1351 } 1352 } else 1353 ret = huge_salloc(ptr); 1354 1355 return (ret); 1356 } 1357 1358 JEMALLOC_ALWAYS_INLINE void 1359 arena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) 1360 { 1361 arena_chunk_t *chunk; 1362 size_t pageind, mapbits; 1363 1364 assert(ptr != NULL); 1365 1366 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1367 if (likely(chunk != ptr)) { 1368 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1369 #if defined(__ANDROID__) 1370 /* Verify the ptr is actually in the chunk. */ 1371 if (unlikely(pageind < map_bias || pageind >= chunk_npages)) { 1372 __libc_fatal("Invalid address %p passed to free: invalid page index", ptr); 1373 } 1374 #endif 1375 mapbits = arena_mapbits_get(chunk, pageind); 1376 assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 1377 #if defined(__ANDROID__) 1378 /* Verify the ptr has been allocated. */ 1379 if (unlikely((mapbits & CHUNK_MAP_ALLOCATED) == 0)) { 1380 __libc_fatal("Invalid address %p passed to free: value not allocated", ptr); 1381 } 1382 #endif 1383 if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) { 1384 /* Small allocation. */ 1385 if (likely(tcache != NULL)) { 1386 szind_t binind = arena_ptr_small_binind_get(ptr, 1387 mapbits); 1388 tcache_dalloc_small(tsd, tcache, ptr, binind, 1389 slow_path); 1390 } else { 1391 arena_dalloc_small(tsd, extent_node_arena_get( 1392 &chunk->node), chunk, ptr, pageind); 1393 } 1394 } else { 1395 size_t size = arena_mapbits_large_size_get(chunk, 1396 pageind); 1397 1398 assert(config_cache_oblivious || ((uintptr_t)ptr & 1399 PAGE_MASK) == 0); 1400 1401 if (likely(tcache != NULL) && size - large_pad <= 1402 tcache_maxclass) { 1403 tcache_dalloc_large(tsd, tcache, ptr, size - 1404 large_pad, slow_path); 1405 } else { 1406 arena_dalloc_large(tsd, extent_node_arena_get( 1407 &chunk->node), chunk, ptr); 1408 } 1409 } 1410 } else 1411 huge_dalloc(tsd, ptr, tcache); 1412 } 1413 1414 JEMALLOC_ALWAYS_INLINE void 1415 arena_sdalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache) 1416 { 1417 arena_chunk_t *chunk; 1418 1419 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1420 if (likely(chunk != ptr)) { 1421 if (config_prof && opt_prof) { 1422 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> 1423 LG_PAGE; 1424 assert(arena_mapbits_allocated_get(chunk, pageind) != 1425 0); 1426 if (arena_mapbits_large_get(chunk, pageind) != 0) { 1427 /* 1428 * Make sure to use promoted size, not request 1429 * size. 1430 */ 1431 size = arena_mapbits_large_size_get(chunk, 1432 pageind) - large_pad; 1433 } 1434 } 1435 assert(s2u(size) == s2u(arena_salloc(ptr, false))); 1436 1437 if (likely(size <= SMALL_MAXCLASS)) { 1438 /* Small allocation. */ 1439 if (likely(tcache != NULL)) { 1440 szind_t binind = size2index(size); 1441 tcache_dalloc_small(tsd, tcache, ptr, binind, 1442 true); 1443 } else { 1444 size_t pageind = ((uintptr_t)ptr - 1445 (uintptr_t)chunk) >> LG_PAGE; 1446 arena_dalloc_small(tsd, extent_node_arena_get( 1447 &chunk->node), chunk, ptr, pageind); 1448 } 1449 } else { 1450 assert(config_cache_oblivious || ((uintptr_t)ptr & 1451 PAGE_MASK) == 0); 1452 1453 if (likely(tcache != NULL) && size <= tcache_maxclass) { 1454 tcache_dalloc_large(tsd, tcache, ptr, size, 1455 true); 1456 } else { 1457 arena_dalloc_large(tsd, extent_node_arena_get( 1458 &chunk->node), chunk, ptr); 1459 } 1460 } 1461 } else 1462 huge_dalloc(tsd, ptr, tcache); 1463 } 1464 # endif /* JEMALLOC_ARENA_INLINE_B */ 1465 #endif 1466 1467 #endif /* JEMALLOC_H_INLINES */ 1468 /******************************************************************************/ 1469
