1 /******************************************************************************/ 2 #ifdef JEMALLOC_H_TYPES 3 4 /* Maximum number of malloc_tsd users with cleanup functions. */ 5 #define MALLOC_TSD_CLEANUPS_MAX 2 6 7 typedef bool (*malloc_tsd_cleanup_t)(void); 8 9 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 10 !defined(_WIN32)) 11 typedef struct tsd_init_block_s tsd_init_block_t; 12 typedef struct tsd_init_head_s tsd_init_head_t; 13 #endif 14 15 typedef struct tsd_s tsd_t; 16 typedef struct tsdn_s tsdn_t; 17 18 #define TSDN_NULL ((tsdn_t *)0) 19 20 typedef enum { 21 tsd_state_uninitialized, 22 tsd_state_nominal, 23 tsd_state_purgatory, 24 tsd_state_reincarnated 25 } tsd_state_t; 26 27 /* 28 * TLS/TSD-agnostic macro-based implementation of thread-specific data. There 29 * are five macros that support (at least) three use cases: file-private, 30 * library-private, and library-private inlined. Following is an example 31 * library-private tsd variable: 32 * 33 * In example.h: 34 * typedef struct { 35 * int x; 36 * int y; 37 * } example_t; 38 * #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0}) 39 * malloc_tsd_types(example_, example_t) 40 * malloc_tsd_protos(, example_, example_t) 41 * malloc_tsd_externs(example_, example_t) 42 * In example.c: 43 * malloc_tsd_data(, example_, example_t, EX_INITIALIZER) 44 * malloc_tsd_funcs(, example_, example_t, EX_INITIALIZER, 45 * example_tsd_cleanup) 46 * 47 * The result is a set of generated functions, e.g.: 48 * 49 * bool example_tsd_boot(void) {...} 50 * bool example_tsd_booted_get(void) {...} 51 * example_t *example_tsd_get(bool init) {...} 52 * void example_tsd_set(example_t *val) {...} 53 * 54 * Note that all of the functions deal in terms of (a_type *) rather than 55 * (a_type) so that it is possible to support non-pointer types (unlike 56 * pthreads TSD). example_tsd_cleanup() is passed an (a_type *) pointer that is 57 * cast to (void *). This means that the cleanup function needs to cast the 58 * function argument to (a_type *), then dereference the resulting pointer to 59 * access fields, e.g. 60 * 61 * void 62 * example_tsd_cleanup(void *arg) 63 * { 64 * example_t *example = (example_t *)arg; 65 * 66 * example->x = 42; 67 * [...] 68 * if ([want the cleanup function to be called again]) 69 * example_tsd_set(example); 70 * } 71 * 72 * If example_tsd_set() is called within example_tsd_cleanup(), it will be 73 * called again. This is similar to how pthreads TSD destruction works, except 74 * that pthreads only calls the cleanup function again if the value was set to 75 * non-NULL. 76 */ 77 78 /* malloc_tsd_types(). */ 79 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 80 #define malloc_tsd_types(a_name, a_type) 81 #elif (defined(JEMALLOC_TLS)) 82 #define malloc_tsd_types(a_name, a_type) 83 #elif (defined(_WIN32)) 84 #define malloc_tsd_types(a_name, a_type) \ 85 typedef struct { \ 86 bool initialized; \ 87 a_type val; \ 88 } a_name##tsd_wrapper_t; 89 #else 90 #define malloc_tsd_types(a_name, a_type) \ 91 typedef struct { \ 92 bool initialized; \ 93 a_type val; \ 94 } a_name##tsd_wrapper_t; 95 #endif 96 97 /* malloc_tsd_protos(). */ 98 #define malloc_tsd_protos(a_attr, a_name, a_type) \ 99 a_attr bool \ 100 a_name##tsd_boot0(void); \ 101 a_attr void \ 102 a_name##tsd_boot1(void); \ 103 a_attr bool \ 104 a_name##tsd_boot(void); \ 105 a_attr bool \ 106 a_name##tsd_booted_get(void); \ 107 a_attr a_type * \ 108 a_name##tsd_get(bool init); \ 109 a_attr void \ 110 a_name##tsd_set(a_type *val); 111 112 /* malloc_tsd_externs(). */ 113 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 114 #define malloc_tsd_externs(a_name, a_type) \ 115 extern __thread a_type a_name##tsd_tls; \ 116 extern __thread bool a_name##tsd_initialized; \ 117 extern bool a_name##tsd_booted; 118 #elif (defined(JEMALLOC_TLS)) 119 #define malloc_tsd_externs(a_name, a_type) \ 120 extern __thread a_type a_name##tsd_tls; \ 121 extern pthread_key_t a_name##tsd_tsd; \ 122 extern bool a_name##tsd_booted; 123 #elif (defined(_WIN32)) 124 #define malloc_tsd_externs(a_name, a_type) \ 125 extern DWORD a_name##tsd_tsd; \ 126 extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \ 127 extern bool a_name##tsd_booted; 128 #else 129 #define malloc_tsd_externs(a_name, a_type) \ 130 extern pthread_key_t a_name##tsd_tsd; \ 131 extern tsd_init_head_t a_name##tsd_init_head; \ 132 extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \ 133 extern bool a_name##tsd_booted; 134 #endif 135 136 /* malloc_tsd_data(). */ 137 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 138 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 139 a_attr __thread a_type JEMALLOC_TLS_MODEL \ 140 a_name##tsd_tls = a_initializer; \ 141 a_attr __thread bool JEMALLOC_TLS_MODEL \ 142 a_name##tsd_initialized = false; \ 143 a_attr bool a_name##tsd_booted = false; 144 #elif (defined(JEMALLOC_TLS)) 145 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 146 a_attr __thread a_type JEMALLOC_TLS_MODEL \ 147 a_name##tsd_tls = a_initializer; \ 148 a_attr pthread_key_t a_name##tsd_tsd; \ 149 a_attr bool a_name##tsd_booted = false; 150 #elif (defined(_WIN32)) 151 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 152 a_attr DWORD a_name##tsd_tsd; \ 153 a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \ 154 false, \ 155 a_initializer \ 156 }; \ 157 a_attr bool a_name##tsd_booted = false; 158 #else 159 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 160 a_attr pthread_key_t a_name##tsd_tsd; \ 161 a_attr tsd_init_head_t a_name##tsd_init_head = { \ 162 ql_head_initializer(blocks), \ 163 MALLOC_MUTEX_INITIALIZER \ 164 }; \ 165 a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \ 166 false, \ 167 a_initializer \ 168 }; \ 169 a_attr bool a_name##tsd_booted = false; 170 #endif 171 172 /* malloc_tsd_funcs(). */ 173 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 174 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 175 a_cleanup) \ 176 /* Initialization/cleanup. */ \ 177 a_attr bool \ 178 a_name##tsd_cleanup_wrapper(void) \ 179 { \ 180 \ 181 if (a_name##tsd_initialized) { \ 182 a_name##tsd_initialized = false; \ 183 a_cleanup(&a_name##tsd_tls); \ 184 } \ 185 return (a_name##tsd_initialized); \ 186 } \ 187 a_attr bool \ 188 a_name##tsd_boot0(void) \ 189 { \ 190 \ 191 if (a_cleanup != malloc_tsd_no_cleanup) { \ 192 malloc_tsd_cleanup_register( \ 193 &a_name##tsd_cleanup_wrapper); \ 194 } \ 195 a_name##tsd_booted = true; \ 196 return (false); \ 197 } \ 198 a_attr void \ 199 a_name##tsd_boot1(void) \ 200 { \ 201 \ 202 /* Do nothing. */ \ 203 } \ 204 a_attr bool \ 205 a_name##tsd_boot(void) \ 206 { \ 207 \ 208 return (a_name##tsd_boot0()); \ 209 } \ 210 a_attr bool \ 211 a_name##tsd_booted_get(void) \ 212 { \ 213 \ 214 return (a_name##tsd_booted); \ 215 } \ 216 a_attr bool \ 217 a_name##tsd_get_allocates(void) \ 218 { \ 219 \ 220 return (false); \ 221 } \ 222 /* Get/set. */ \ 223 a_attr a_type * \ 224 a_name##tsd_get(bool init) \ 225 { \ 226 \ 227 assert(a_name##tsd_booted); \ 228 return (&a_name##tsd_tls); \ 229 } \ 230 a_attr void \ 231 a_name##tsd_set(a_type *val) \ 232 { \ 233 \ 234 assert(a_name##tsd_booted); \ 235 a_name##tsd_tls = (*val); \ 236 if (a_cleanup != malloc_tsd_no_cleanup) \ 237 a_name##tsd_initialized = true; \ 238 } 239 #elif (defined(JEMALLOC_TLS)) 240 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 241 a_cleanup) \ 242 /* Initialization/cleanup. */ \ 243 a_attr bool \ 244 a_name##tsd_boot0(void) \ 245 { \ 246 \ 247 if (a_cleanup != malloc_tsd_no_cleanup) { \ 248 if (pthread_key_create(&a_name##tsd_tsd, a_cleanup) != \ 249 0) \ 250 return (true); \ 251 } \ 252 a_name##tsd_booted = true; \ 253 return (false); \ 254 } \ 255 a_attr void \ 256 a_name##tsd_boot1(void) \ 257 { \ 258 \ 259 /* Do nothing. */ \ 260 } \ 261 a_attr bool \ 262 a_name##tsd_boot(void) \ 263 { \ 264 \ 265 return (a_name##tsd_boot0()); \ 266 } \ 267 a_attr bool \ 268 a_name##tsd_booted_get(void) \ 269 { \ 270 \ 271 return (a_name##tsd_booted); \ 272 } \ 273 a_attr bool \ 274 a_name##tsd_get_allocates(void) \ 275 { \ 276 \ 277 return (false); \ 278 } \ 279 /* Get/set. */ \ 280 a_attr a_type * \ 281 a_name##tsd_get(bool init) \ 282 { \ 283 \ 284 assert(a_name##tsd_booted); \ 285 return (&a_name##tsd_tls); \ 286 } \ 287 a_attr void \ 288 a_name##tsd_set(a_type *val) \ 289 { \ 290 \ 291 assert(a_name##tsd_booted); \ 292 a_name##tsd_tls = (*val); \ 293 if (a_cleanup != malloc_tsd_no_cleanup) { \ 294 if (pthread_setspecific(a_name##tsd_tsd, \ 295 (void *)(&a_name##tsd_tls))) { \ 296 malloc_write("<jemalloc>: Error" \ 297 " setting TSD for "#a_name"\n"); \ 298 if (opt_abort) \ 299 abort(); \ 300 } \ 301 } \ 302 } 303 #elif (defined(_WIN32)) 304 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 305 a_cleanup) \ 306 /* Initialization/cleanup. */ \ 307 a_attr bool \ 308 a_name##tsd_cleanup_wrapper(void) \ 309 { \ 310 DWORD error = GetLastError(); \ 311 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ 312 TlsGetValue(a_name##tsd_tsd); \ 313 SetLastError(error); \ 314 \ 315 if (wrapper == NULL) \ 316 return (false); \ 317 if (a_cleanup != malloc_tsd_no_cleanup && \ 318 wrapper->initialized) { \ 319 wrapper->initialized = false; \ 320 a_cleanup(&wrapper->val); \ 321 if (wrapper->initialized) { \ 322 /* Trigger another cleanup round. */ \ 323 return (true); \ 324 } \ 325 } \ 326 malloc_tsd_dalloc(wrapper); \ 327 return (false); \ 328 } \ 329 a_attr void \ 330 a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \ 331 { \ 332 \ 333 if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) { \ 334 malloc_write("<jemalloc>: Error setting" \ 335 " TSD for "#a_name"\n"); \ 336 abort(); \ 337 } \ 338 } \ 339 a_attr a_name##tsd_wrapper_t * \ 340 a_name##tsd_wrapper_get(bool init) \ 341 { \ 342 DWORD error = GetLastError(); \ 343 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ 344 TlsGetValue(a_name##tsd_tsd); \ 345 SetLastError(error); \ 346 \ 347 if (init && unlikely(wrapper == NULL)) { \ 348 wrapper = (a_name##tsd_wrapper_t *) \ 349 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 350 if (wrapper == NULL) { \ 351 malloc_write("<jemalloc>: Error allocating" \ 352 " TSD for "#a_name"\n"); \ 353 abort(); \ 354 } else { \ 355 wrapper->initialized = false; \ 356 wrapper->val = a_initializer; \ 357 } \ 358 a_name##tsd_wrapper_set(wrapper); \ 359 } \ 360 return (wrapper); \ 361 } \ 362 a_attr bool \ 363 a_name##tsd_boot0(void) \ 364 { \ 365 \ 366 a_name##tsd_tsd = TlsAlloc(); \ 367 if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES) \ 368 return (true); \ 369 if (a_cleanup != malloc_tsd_no_cleanup) { \ 370 malloc_tsd_cleanup_register( \ 371 &a_name##tsd_cleanup_wrapper); \ 372 } \ 373 a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \ 374 a_name##tsd_booted = true; \ 375 return (false); \ 376 } \ 377 a_attr void \ 378 a_name##tsd_boot1(void) \ 379 { \ 380 a_name##tsd_wrapper_t *wrapper; \ 381 wrapper = (a_name##tsd_wrapper_t *) \ 382 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 383 if (wrapper == NULL) { \ 384 malloc_write("<jemalloc>: Error allocating" \ 385 " TSD for "#a_name"\n"); \ 386 abort(); \ 387 } \ 388 memcpy(wrapper, &a_name##tsd_boot_wrapper, \ 389 sizeof(a_name##tsd_wrapper_t)); \ 390 a_name##tsd_wrapper_set(wrapper); \ 391 } \ 392 a_attr bool \ 393 a_name##tsd_boot(void) \ 394 { \ 395 \ 396 if (a_name##tsd_boot0()) \ 397 return (true); \ 398 a_name##tsd_boot1(); \ 399 return (false); \ 400 } \ 401 a_attr bool \ 402 a_name##tsd_booted_get(void) \ 403 { \ 404 \ 405 return (a_name##tsd_booted); \ 406 } \ 407 a_attr bool \ 408 a_name##tsd_get_allocates(void) \ 409 { \ 410 \ 411 return (true); \ 412 } \ 413 /* Get/set. */ \ 414 a_attr a_type * \ 415 a_name##tsd_get(bool init) \ 416 { \ 417 a_name##tsd_wrapper_t *wrapper; \ 418 \ 419 assert(a_name##tsd_booted); \ 420 wrapper = a_name##tsd_wrapper_get(init); \ 421 if (a_name##tsd_get_allocates() && !init && wrapper == NULL) \ 422 return (NULL); \ 423 return (&wrapper->val); \ 424 } \ 425 a_attr void \ 426 a_name##tsd_set(a_type *val) \ 427 { \ 428 a_name##tsd_wrapper_t *wrapper; \ 429 \ 430 assert(a_name##tsd_booted); \ 431 wrapper = a_name##tsd_wrapper_get(true); \ 432 wrapper->val = *(val); \ 433 if (a_cleanup != malloc_tsd_no_cleanup) \ 434 wrapper->initialized = true; \ 435 } 436 #else 437 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 438 a_cleanup) \ 439 /* Initialization/cleanup. */ \ 440 a_attr void \ 441 a_name##tsd_cleanup_wrapper(void *arg) \ 442 { \ 443 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)arg; \ 444 \ 445 if (a_cleanup != malloc_tsd_no_cleanup && \ 446 wrapper->initialized) { \ 447 wrapper->initialized = false; \ 448 a_cleanup(&wrapper->val); \ 449 if (wrapper->initialized) { \ 450 /* Trigger another cleanup round. */ \ 451 if (pthread_setspecific(a_name##tsd_tsd, \ 452 (void *)wrapper)) { \ 453 malloc_write("<jemalloc>: Error" \ 454 " setting TSD for "#a_name"\n"); \ 455 if (opt_abort) \ 456 abort(); \ 457 } \ 458 return; \ 459 } \ 460 } \ 461 malloc_tsd_dalloc(wrapper); \ 462 } \ 463 a_attr void \ 464 a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \ 465 { \ 466 \ 467 if (pthread_setspecific(a_name##tsd_tsd, \ 468 (void *)wrapper)) { \ 469 malloc_write("<jemalloc>: Error setting" \ 470 " TSD for "#a_name"\n"); \ 471 abort(); \ 472 } \ 473 } \ 474 a_attr a_name##tsd_wrapper_t * \ 475 a_name##tsd_wrapper_get(bool init) \ 476 { \ 477 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ 478 pthread_getspecific(a_name##tsd_tsd); \ 479 \ 480 if (init && unlikely(wrapper == NULL)) { \ 481 tsd_init_block_t block; \ 482 wrapper = tsd_init_check_recursion( \ 483 &a_name##tsd_init_head, &block); \ 484 if (wrapper) \ 485 return (wrapper); \ 486 wrapper = (a_name##tsd_wrapper_t *) \ 487 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 488 block.data = wrapper; \ 489 if (wrapper == NULL) { \ 490 malloc_write("<jemalloc>: Error allocating" \ 491 " TSD for "#a_name"\n"); \ 492 abort(); \ 493 } else { \ 494 wrapper->initialized = false; \ 495 wrapper->val = a_initializer; \ 496 } \ 497 a_name##tsd_wrapper_set(wrapper); \ 498 tsd_init_finish(&a_name##tsd_init_head, &block); \ 499 } \ 500 return (wrapper); \ 501 } \ 502 a_attr bool \ 503 a_name##tsd_boot0(void) \ 504 { \ 505 \ 506 if (pthread_key_create(&a_name##tsd_tsd, \ 507 a_name##tsd_cleanup_wrapper) != 0) \ 508 return (true); \ 509 a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \ 510 a_name##tsd_booted = true; \ 511 return (false); \ 512 } \ 513 a_attr void \ 514 a_name##tsd_boot1(void) \ 515 { \ 516 a_name##tsd_wrapper_t *wrapper; \ 517 wrapper = (a_name##tsd_wrapper_t *) \ 518 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 519 if (wrapper == NULL) { \ 520 malloc_write("<jemalloc>: Error allocating" \ 521 " TSD for "#a_name"\n"); \ 522 abort(); \ 523 } \ 524 memcpy(wrapper, &a_name##tsd_boot_wrapper, \ 525 sizeof(a_name##tsd_wrapper_t)); \ 526 a_name##tsd_wrapper_set(wrapper); \ 527 } \ 528 a_attr bool \ 529 a_name##tsd_boot(void) \ 530 { \ 531 \ 532 if (a_name##tsd_boot0()) \ 533 return (true); \ 534 a_name##tsd_boot1(); \ 535 return (false); \ 536 } \ 537 a_attr bool \ 538 a_name##tsd_booted_get(void) \ 539 { \ 540 \ 541 return (a_name##tsd_booted); \ 542 } \ 543 a_attr bool \ 544 a_name##tsd_get_allocates(void) \ 545 { \ 546 \ 547 return (true); \ 548 } \ 549 /* Get/set. */ \ 550 a_attr a_type * \ 551 a_name##tsd_get(bool init) \ 552 { \ 553 a_name##tsd_wrapper_t *wrapper; \ 554 \ 555 assert(a_name##tsd_booted); \ 556 wrapper = a_name##tsd_wrapper_get(init); \ 557 if (a_name##tsd_get_allocates() && !init && wrapper == NULL) \ 558 return (NULL); \ 559 return (&wrapper->val); \ 560 } \ 561 a_attr void \ 562 a_name##tsd_set(a_type *val) \ 563 { \ 564 a_name##tsd_wrapper_t *wrapper; \ 565 \ 566 assert(a_name##tsd_booted); \ 567 wrapper = a_name##tsd_wrapper_get(true); \ 568 wrapper->val = *(val); \ 569 if (a_cleanup != malloc_tsd_no_cleanup) \ 570 wrapper->initialized = true; \ 571 } 572 #endif 573 574 #endif /* JEMALLOC_H_TYPES */ 575 /******************************************************************************/ 576 #ifdef JEMALLOC_H_STRUCTS 577 578 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 579 !defined(_WIN32)) 580 struct tsd_init_block_s { 581 ql_elm(tsd_init_block_t) link; 582 pthread_t thread; 583 void *data; 584 }; 585 struct tsd_init_head_s { 586 ql_head(tsd_init_block_t) blocks; 587 malloc_mutex_t lock; 588 }; 589 #endif 590 591 #define MALLOC_TSD \ 592 /* O(name, type) */ \ 593 O(tcache, tcache_t *) \ 594 O(thread_allocated, uint64_t) \ 595 O(thread_deallocated, uint64_t) \ 596 O(prof_tdata, prof_tdata_t *) \ 597 O(iarena, arena_t *) \ 598 O(arena, arena_t *) \ 599 O(arenas_tdata, arena_tdata_t *) \ 600 O(narenas_tdata, unsigned) \ 601 O(arenas_tdata_bypass, bool) \ 602 O(tcache_enabled, tcache_enabled_t) \ 603 O(quarantine, quarantine_t *) \ 604 O(witnesses, witness_list_t) \ 605 O(witness_fork, bool) \ 606 607 #define TSD_INITIALIZER { \ 608 tsd_state_uninitialized, \ 609 NULL, \ 610 0, \ 611 0, \ 612 NULL, \ 613 NULL, \ 614 NULL, \ 615 NULL, \ 616 0, \ 617 false, \ 618 tcache_enabled_default, \ 619 NULL, \ 620 ql_head_initializer(witnesses), \ 621 false \ 622 } 623 624 struct tsd_s { 625 tsd_state_t state; 626 #define O(n, t) \ 627 t n; 628 MALLOC_TSD 629 #undef O 630 }; 631 632 /* 633 * Wrapper around tsd_t that makes it possible to avoid implicit conversion 634 * between tsd_t and tsdn_t, where tsdn_t is "nullable" and has to be 635 * explicitly converted to tsd_t, which is non-nullable. 636 */ 637 struct tsdn_s { 638 tsd_t tsd; 639 }; 640 641 static const tsd_t tsd_initializer = TSD_INITIALIZER; 642 643 malloc_tsd_types(, tsd_t) 644 645 #endif /* JEMALLOC_H_STRUCTS */ 646 /******************************************************************************/ 647 #ifdef JEMALLOC_H_EXTERNS 648 649 void *malloc_tsd_malloc(size_t size); 650 void malloc_tsd_dalloc(void *wrapper); 651 void malloc_tsd_no_cleanup(void *arg); 652 void malloc_tsd_cleanup_register(bool (*f)(void)); 653 tsd_t *malloc_tsd_boot0(void); 654 void malloc_tsd_boot1(void); 655 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 656 !defined(_WIN32)) 657 void *tsd_init_check_recursion(tsd_init_head_t *head, 658 tsd_init_block_t *block); 659 void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block); 660 #endif 661 void tsd_cleanup(void *arg); 662 663 #endif /* JEMALLOC_H_EXTERNS */ 664 /******************************************************************************/ 665 #ifdef JEMALLOC_H_INLINES 666 667 #ifndef JEMALLOC_ENABLE_INLINE 668 malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t) 669 670 tsd_t *tsd_fetch_impl(bool init); 671 tsd_t *tsd_fetch(void); 672 tsdn_t *tsd_tsdn(tsd_t *tsd); 673 bool tsd_nominal(tsd_t *tsd); 674 #define O(n, t) \ 675 t *tsd_##n##p_get(tsd_t *tsd); \ 676 t tsd_##n##_get(tsd_t *tsd); \ 677 void tsd_##n##_set(tsd_t *tsd, t n); 678 MALLOC_TSD 679 #undef O 680 tsdn_t *tsdn_fetch(void); 681 bool tsdn_null(const tsdn_t *tsdn); 682 tsd_t *tsdn_tsd(tsdn_t *tsdn); 683 #endif 684 685 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_)) 686 malloc_tsd_externs(, tsd_t) 687 malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup) 688 689 JEMALLOC_ALWAYS_INLINE tsd_t * 690 tsd_fetch_impl(bool init) 691 { 692 tsd_t *tsd = tsd_get(init); 693 694 if (!init && tsd_get_allocates() && tsd == NULL) 695 return (NULL); 696 assert(tsd != NULL); 697 698 if (unlikely(tsd->state != tsd_state_nominal)) { 699 if (tsd->state == tsd_state_uninitialized) { 700 tsd->state = tsd_state_nominal; 701 /* Trigger cleanup handler registration. */ 702 tsd_set(tsd); 703 } else if (tsd->state == tsd_state_purgatory) { 704 tsd->state = tsd_state_reincarnated; 705 tsd_set(tsd); 706 } else 707 assert(tsd->state == tsd_state_reincarnated); 708 } 709 710 return (tsd); 711 } 712 713 JEMALLOC_ALWAYS_INLINE tsd_t * 714 tsd_fetch(void) 715 { 716 717 return (tsd_fetch_impl(true)); 718 } 719 720 JEMALLOC_ALWAYS_INLINE tsdn_t * 721 tsd_tsdn(tsd_t *tsd) 722 { 723 724 return ((tsdn_t *)tsd); 725 } 726 727 JEMALLOC_INLINE bool 728 tsd_nominal(tsd_t *tsd) 729 { 730 731 return (tsd->state == tsd_state_nominal); 732 } 733 734 #define O(n, t) \ 735 JEMALLOC_ALWAYS_INLINE t * \ 736 tsd_##n##p_get(tsd_t *tsd) \ 737 { \ 738 \ 739 return (&tsd->n); \ 740 } \ 741 \ 742 JEMALLOC_ALWAYS_INLINE t \ 743 tsd_##n##_get(tsd_t *tsd) \ 744 { \ 745 \ 746 return (*tsd_##n##p_get(tsd)); \ 747 } \ 748 \ 749 JEMALLOC_ALWAYS_INLINE void \ 750 tsd_##n##_set(tsd_t *tsd, t n) \ 751 { \ 752 \ 753 assert(tsd->state == tsd_state_nominal); \ 754 tsd->n = n; \ 755 } 756 MALLOC_TSD 757 #undef O 758 759 JEMALLOC_ALWAYS_INLINE tsdn_t * 760 tsdn_fetch(void) 761 { 762 763 if (!tsd_booted_get()) 764 return (NULL); 765 766 return (tsd_tsdn(tsd_fetch_impl(false))); 767 } 768 769 JEMALLOC_ALWAYS_INLINE bool 770 tsdn_null(const tsdn_t *tsdn) 771 { 772 773 return (tsdn == NULL); 774 } 775 776 JEMALLOC_ALWAYS_INLINE tsd_t * 777 tsdn_tsd(tsdn_t *tsdn) 778 { 779 780 assert(!tsdn_null(tsdn)); 781 782 return (&tsdn->tsd); 783 } 784 #endif 785 786 #endif /* JEMALLOC_H_INLINES */ 787 /******************************************************************************/ 788
