1 /******************************************************************************/ 2 #ifdef JEMALLOC_H_TYPES 3 4 typedef struct tcache_bin_info_s tcache_bin_info_t; 5 typedef struct tcache_bin_s tcache_bin_t; 6 typedef struct tcache_s tcache_t; 7 typedef struct tcaches_s tcaches_t; 8 9 /* 10 * tcache pointers close to NULL are used to encode state information that is 11 * used for two purposes: preventing thread caching on a per thread basis and 12 * cleaning up during thread shutdown. 13 */ 14 #define TCACHE_STATE_DISABLED ((tcache_t *)(uintptr_t)1) 15 #define TCACHE_STATE_REINCARNATED ((tcache_t *)(uintptr_t)2) 16 #define TCACHE_STATE_PURGATORY ((tcache_t *)(uintptr_t)3) 17 #define TCACHE_STATE_MAX TCACHE_STATE_PURGATORY 18 19 /* 20 * Absolute minimum number of cache slots for each small bin. 21 */ 22 #define TCACHE_NSLOTS_SMALL_MIN 20 23 24 /* 25 * Absolute maximum number of cache slots for each small bin in the thread 26 * cache. This is an additional constraint beyond that imposed as: twice the 27 * number of regions per run for this size class. 28 * 29 * This constant must be an even number. 30 */ 31 #if defined(ANDROID_TCACHE_NSLOTS_SMALL_MAX) 32 #define TCACHE_NSLOTS_SMALL_MAX ANDROID_TCACHE_NSLOTS_SMALL_MAX 33 #else 34 #define TCACHE_NSLOTS_SMALL_MAX 200 35 #endif 36 37 /* Number of cache slots for large size classes. */ 38 #if defined(ANDROID_TCACHE_NSLOTS_LARGE) 39 #define TCACHE_NSLOTS_LARGE ANDROID_TCACHE_NSLOTS_LARGE 40 #else 41 #define TCACHE_NSLOTS_LARGE 20 42 #endif 43 44 /* (1U << opt_lg_tcache_max) is used to compute tcache_maxclass. */ 45 #if defined(ANDROID_LG_TCACHE_MAXCLASS_DEFAULT) 46 #define LG_TCACHE_MAXCLASS_DEFAULT ANDROID_LG_TCACHE_MAXCLASS_DEFAULT 47 #else 48 #define LG_TCACHE_MAXCLASS_DEFAULT 15 49 #endif 50 51 /* 52 * TCACHE_GC_SWEEP is the approximate number of allocation events between 53 * full GC sweeps. Integer rounding may cause the actual number to be 54 * slightly higher, since GC is performed incrementally. 55 */ 56 #define TCACHE_GC_SWEEP 8192 57 58 /* Number of tcache allocation/deallocation events between incremental GCs. */ 59 #define TCACHE_GC_INCR \ 60 ((TCACHE_GC_SWEEP / NBINS) + ((TCACHE_GC_SWEEP / NBINS == 0) ? 0 : 1)) 61 62 #endif /* JEMALLOC_H_TYPES */ 63 /******************************************************************************/ 64 #ifdef JEMALLOC_H_STRUCTS 65 66 typedef enum { 67 tcache_enabled_false = 0, /* Enable cast to/from bool. */ 68 tcache_enabled_true = 1, 69 tcache_enabled_default = 2 70 } tcache_enabled_t; 71 72 /* 73 * Read-only information associated with each element of tcache_t's tbins array 74 * is stored separately, mainly to reduce memory usage. 75 */ 76 struct tcache_bin_info_s { 77 unsigned ncached_max; /* Upper limit on ncached. */ 78 }; 79 80 struct tcache_bin_s { 81 tcache_bin_stats_t tstats; 82 int low_water; /* Min # cached since last GC. */ 83 unsigned lg_fill_div; /* Fill (ncached_max >> lg_fill_div). */ 84 unsigned ncached; /* # of cached objects. */ 85 /* 86 * To make use of adjacent cacheline prefetch, the items in the avail 87 * stack goes to higher address for newer allocations. avail points 88 * just above the available space, which means that 89 * avail[-ncached, ... -1] are available items and the lowest item will 90 * be allocated first. 91 */ 92 void **avail; /* Stack of available objects. */ 93 }; 94 95 struct tcache_s { 96 ql_elm(tcache_t) link; /* Used for aggregating stats. */ 97 uint64_t prof_accumbytes;/* Cleared after arena_prof_accum(). */ 98 ticker_t gc_ticker; /* Drives incremental GC. */ 99 szind_t next_gc_bin; /* Next bin to GC. */ 100 tcache_bin_t tbins[1]; /* Dynamically sized. */ 101 /* 102 * The pointer stacks associated with tbins follow as a contiguous 103 * array. During tcache initialization, the avail pointer in each 104 * element of tbins is initialized to point to the proper offset within 105 * this array. 106 */ 107 }; 108 109 /* Linkage for list of available (previously used) explicit tcache IDs. */ 110 struct tcaches_s { 111 union { 112 tcache_t *tcache; 113 tcaches_t *next; 114 }; 115 }; 116 117 #endif /* JEMALLOC_H_STRUCTS */ 118 /******************************************************************************/ 119 #ifdef JEMALLOC_H_EXTERNS 120 121 extern bool opt_tcache; 122 extern ssize_t opt_lg_tcache_max; 123 124 extern tcache_bin_info_t *tcache_bin_info; 125 126 /* 127 * Number of tcache bins. There are NBINS small-object bins, plus 0 or more 128 * large-object bins. 129 */ 130 extern unsigned nhbins; 131 132 /* Maximum cached size class. */ 133 extern size_t tcache_maxclass; 134 135 /* 136 * Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and 137 * usable via the MALLOCX_TCACHE() flag. The automatic per thread tcaches are 138 * completely disjoint from this data structure. tcaches starts off as a sparse 139 * array, so it has no physical memory footprint until individual pages are 140 * touched. This allows the entire array to be allocated the first time an 141 * explicit tcache is created without a disproportionate impact on memory usage. 142 */ 143 extern tcaches_t *tcaches; 144 145 size_t tcache_salloc(tsdn_t *tsdn, const void *ptr); 146 void tcache_event_hard(tsd_t *tsd, tcache_t *tcache); 147 void *tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, 148 tcache_bin_t *tbin, szind_t binind, bool *tcache_success); 149 void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, 150 szind_t binind, unsigned rem); 151 void tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind, 152 unsigned rem, tcache_t *tcache); 153 void tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, 154 arena_t *oldarena, arena_t *newarena); 155 tcache_t *tcache_get_hard(tsd_t *tsd); 156 tcache_t *tcache_create(tsdn_t *tsdn, arena_t *arena); 157 void tcache_cleanup(tsd_t *tsd); 158 void tcache_enabled_cleanup(tsd_t *tsd); 159 void tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena); 160 bool tcaches_create(tsd_t *tsd, unsigned *r_ind); 161 void tcaches_flush(tsd_t *tsd, unsigned ind); 162 void tcaches_destroy(tsd_t *tsd, unsigned ind); 163 bool tcache_boot(tsdn_t *tsdn); 164 void tcache_prefork(tsdn_t *tsdn); 165 void tcache_postfork_parent(tsdn_t *tsdn); 166 void tcache_postfork_child(tsdn_t *tsdn); 167 168 #endif /* JEMALLOC_H_EXTERNS */ 169 /******************************************************************************/ 170 #ifdef JEMALLOC_H_INLINES 171 172 #ifndef JEMALLOC_ENABLE_INLINE 173 void tcache_event(tsd_t *tsd, tcache_t *tcache); 174 void tcache_flush(void); 175 bool tcache_enabled_get(void); 176 tcache_t *tcache_get(tsd_t *tsd, bool create); 177 void tcache_enabled_set(bool enabled); 178 void *tcache_alloc_easy(tcache_bin_t *tbin, bool *tcache_success); 179 void *tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, 180 size_t size, szind_t ind, bool zero, bool slow_path); 181 void *tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, 182 size_t size, szind_t ind, bool zero, bool slow_path); 183 void tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, 184 szind_t binind, bool slow_path); 185 void tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, 186 size_t size, bool slow_path); 187 tcache_t *tcaches_get(tsd_t *tsd, unsigned ind); 188 #endif 189 190 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_)) 191 JEMALLOC_INLINE void 192 tcache_flush(void) 193 { 194 tsd_t *tsd; 195 196 cassert(config_tcache); 197 198 tsd = tsd_fetch(); 199 tcache_cleanup(tsd); 200 } 201 202 JEMALLOC_INLINE bool 203 tcache_enabled_get(void) 204 { 205 tsd_t *tsd; 206 tcache_enabled_t tcache_enabled; 207 208 cassert(config_tcache); 209 210 tsd = tsd_fetch(); 211 tcache_enabled = tsd_tcache_enabled_get(tsd); 212 if (tcache_enabled == tcache_enabled_default) { 213 tcache_enabled = (tcache_enabled_t)opt_tcache; 214 tsd_tcache_enabled_set(tsd, tcache_enabled); 215 } 216 217 return ((bool)tcache_enabled); 218 } 219 220 JEMALLOC_INLINE void 221 tcache_enabled_set(bool enabled) 222 { 223 tsd_t *tsd; 224 tcache_enabled_t tcache_enabled; 225 226 cassert(config_tcache); 227 228 tsd = tsd_fetch(); 229 230 tcache_enabled = (tcache_enabled_t)enabled; 231 tsd_tcache_enabled_set(tsd, tcache_enabled); 232 233 if (!enabled) 234 tcache_cleanup(tsd); 235 } 236 237 JEMALLOC_ALWAYS_INLINE tcache_t * 238 tcache_get(tsd_t *tsd, bool create) 239 { 240 tcache_t *tcache; 241 242 if (!config_tcache) 243 return (NULL); 244 245 tcache = tsd_tcache_get(tsd); 246 if (!create) 247 return (tcache); 248 if (unlikely(tcache == NULL) && tsd_nominal(tsd)) { 249 tcache = tcache_get_hard(tsd); 250 tsd_tcache_set(tsd, tcache); 251 } 252 253 return (tcache); 254 } 255 256 JEMALLOC_ALWAYS_INLINE void 257 tcache_event(tsd_t *tsd, tcache_t *tcache) 258 { 259 260 if (TCACHE_GC_INCR == 0) 261 return; 262 263 if (unlikely(ticker_tick(&tcache->gc_ticker))) 264 tcache_event_hard(tsd, tcache); 265 } 266 267 JEMALLOC_ALWAYS_INLINE void * 268 tcache_alloc_easy(tcache_bin_t *tbin, bool *tcache_success) 269 { 270 void *ret; 271 272 if (unlikely(tbin->ncached == 0)) { 273 tbin->low_water = -1; 274 *tcache_success = false; 275 return (NULL); 276 } 277 /* 278 * tcache_success (instead of ret) should be checked upon the return of 279 * this function. We avoid checking (ret == NULL) because there is 280 * never a null stored on the avail stack (which is unknown to the 281 * compiler), and eagerly checking ret would cause pipeline stall 282 * (waiting for the cacheline). 283 */ 284 *tcache_success = true; 285 ret = *(tbin->avail - tbin->ncached); 286 tbin->ncached--; 287 288 if (unlikely((int)tbin->ncached < tbin->low_water)) 289 tbin->low_water = tbin->ncached; 290 291 return (ret); 292 } 293 294 JEMALLOC_ALWAYS_INLINE void * 295 tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, 296 szind_t binind, bool zero, bool slow_path) 297 { 298 void *ret; 299 tcache_bin_t *tbin; 300 bool tcache_success; 301 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 302 303 assert(binind < NBINS); 304 tbin = &tcache->tbins[binind]; 305 ret = tcache_alloc_easy(tbin, &tcache_success); 306 assert(tcache_success == (ret != NULL)); 307 if (unlikely(!tcache_success)) { 308 bool tcache_hard_success; 309 arena = arena_choose(tsd, arena); 310 if (unlikely(arena == NULL)) 311 return (NULL); 312 313 ret = tcache_alloc_small_hard(tsd_tsdn(tsd), arena, tcache, 314 tbin, binind, &tcache_hard_success); 315 if (tcache_hard_success == false) 316 return (NULL); 317 } 318 319 assert(ret); 320 /* 321 * Only compute usize if required. The checks in the following if 322 * statement are all static. 323 */ 324 if (config_prof || (slow_path && config_fill) || unlikely(zero)) { 325 usize = index2size(binind); 326 assert(tcache_salloc(tsd_tsdn(tsd), ret) == usize); 327 } 328 329 if (likely(!zero)) { 330 if (slow_path && config_fill) { 331 if (unlikely(opt_junk_alloc)) { 332 arena_alloc_junk_small(ret, 333 &arena_bin_info[binind], false); 334 } else if (unlikely(opt_zero)) 335 memset(ret, 0, usize); 336 } 337 } else { 338 if (slow_path && config_fill && unlikely(opt_junk_alloc)) { 339 arena_alloc_junk_small(ret, &arena_bin_info[binind], 340 true); 341 } 342 memset(ret, 0, usize); 343 } 344 345 if (config_stats) 346 tbin->tstats.nrequests++; 347 if (config_prof) 348 tcache->prof_accumbytes += usize; 349 tcache_event(tsd, tcache); 350 return (ret); 351 } 352 353 JEMALLOC_ALWAYS_INLINE void * 354 tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, 355 szind_t binind, bool zero, bool slow_path) 356 { 357 void *ret; 358 tcache_bin_t *tbin; 359 bool tcache_success; 360 361 assert(binind < nhbins); 362 tbin = &tcache->tbins[binind]; 363 ret = tcache_alloc_easy(tbin, &tcache_success); 364 assert(tcache_success == (ret != NULL)); 365 if (unlikely(!tcache_success)) { 366 /* 367 * Only allocate one large object at a time, because it's quite 368 * expensive to create one and not use it. 369 */ 370 arena = arena_choose(tsd, arena); 371 if (unlikely(arena == NULL)) 372 return (NULL); 373 374 ret = arena_malloc_large(tsd_tsdn(tsd), arena, binind, zero); 375 if (ret == NULL) 376 return (NULL); 377 } else { 378 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 379 380 /* Only compute usize on demand */ 381 if (config_prof || (slow_path && config_fill) || 382 unlikely(zero)) { 383 usize = index2size(binind); 384 assert(usize <= tcache_maxclass); 385 } 386 387 if (config_prof && usize == LARGE_MINCLASS) { 388 arena_chunk_t *chunk = 389 (arena_chunk_t *)CHUNK_ADDR2BASE(ret); 390 size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >> 391 LG_PAGE); 392 arena_mapbits_large_binind_set(chunk, pageind, 393 BININD_INVALID); 394 } 395 if (likely(!zero)) { 396 if (slow_path && config_fill) { 397 if (unlikely(opt_junk_alloc)) { 398 memset(ret, JEMALLOC_ALLOC_JUNK, 399 usize); 400 } else if (unlikely(opt_zero)) 401 memset(ret, 0, usize); 402 } 403 } else 404 memset(ret, 0, usize); 405 406 if (config_stats) 407 tbin->tstats.nrequests++; 408 if (config_prof) 409 tcache->prof_accumbytes += usize; 410 } 411 412 tcache_event(tsd, tcache); 413 return (ret); 414 } 415 416 JEMALLOC_ALWAYS_INLINE void 417 tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, 418 bool slow_path) 419 { 420 tcache_bin_t *tbin; 421 tcache_bin_info_t *tbin_info; 422 423 assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= SMALL_MAXCLASS); 424 425 if (slow_path && config_fill && unlikely(opt_junk_free)) 426 arena_dalloc_junk_small(ptr, &arena_bin_info[binind]); 427 428 tbin = &tcache->tbins[binind]; 429 tbin_info = &tcache_bin_info[binind]; 430 if (unlikely(tbin->ncached == tbin_info->ncached_max)) { 431 tcache_bin_flush_small(tsd, tcache, tbin, binind, 432 (tbin_info->ncached_max >> 1)); 433 } 434 assert(tbin->ncached < tbin_info->ncached_max); 435 tbin->ncached++; 436 *(tbin->avail - tbin->ncached) = ptr; 437 438 tcache_event(tsd, tcache); 439 } 440 441 JEMALLOC_ALWAYS_INLINE void 442 tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, size_t size, 443 bool slow_path) 444 { 445 szind_t binind; 446 tcache_bin_t *tbin; 447 tcache_bin_info_t *tbin_info; 448 449 assert((size & PAGE_MASK) == 0); 450 assert(tcache_salloc(tsd_tsdn(tsd), ptr) > SMALL_MAXCLASS); 451 assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= tcache_maxclass); 452 453 binind = size2index(size); 454 455 if (slow_path && config_fill && unlikely(opt_junk_free)) 456 arena_dalloc_junk_large(ptr, size); 457 458 tbin = &tcache->tbins[binind]; 459 tbin_info = &tcache_bin_info[binind]; 460 if (unlikely(tbin->ncached == tbin_info->ncached_max)) { 461 tcache_bin_flush_large(tsd, tbin, binind, 462 (tbin_info->ncached_max >> 1), tcache); 463 } 464 assert(tbin->ncached < tbin_info->ncached_max); 465 tbin->ncached++; 466 *(tbin->avail - tbin->ncached) = ptr; 467 468 tcache_event(tsd, tcache); 469 } 470 471 JEMALLOC_ALWAYS_INLINE tcache_t * 472 tcaches_get(tsd_t *tsd, unsigned ind) 473 { 474 tcaches_t *elm = &tcaches[ind]; 475 if (unlikely(elm->tcache == NULL)) { 476 elm->tcache = tcache_create(tsd_tsdn(tsd), arena_choose(tsd, 477 NULL)); 478 } 479 return (elm->tcache); 480 } 481 #endif 482 483 #endif /* JEMALLOC_H_INLINES */ 484 /******************************************************************************/ 485
