1 /******************************************************************************/ 2 #ifdef JEMALLOC_H_TYPES 3 4 /* Maximum number of malloc_tsd users with cleanup functions. */ 5 #define MALLOC_TSD_CLEANUPS_MAX 2 6 7 typedef bool (*malloc_tsd_cleanup_t)(void); 8 9 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 10 !defined(_WIN32)) 11 typedef struct tsd_init_block_s tsd_init_block_t; 12 typedef struct tsd_init_head_s tsd_init_head_t; 13 #endif 14 15 typedef struct tsd_s tsd_t; 16 17 typedef enum { 18 tsd_state_uninitialized, 19 tsd_state_nominal, 20 tsd_state_purgatory, 21 tsd_state_reincarnated 22 } tsd_state_t; 23 24 /* 25 * TLS/TSD-agnostic macro-based implementation of thread-specific data. There 26 * are five macros that support (at least) three use cases: file-private, 27 * library-private, and library-private inlined. Following is an example 28 * library-private tsd variable: 29 * 30 * In example.h: 31 * typedef struct { 32 * int x; 33 * int y; 34 * } example_t; 35 * #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0}) 36 * malloc_tsd_types(example_, example_t) 37 * malloc_tsd_protos(, example_, example_t) 38 * malloc_tsd_externs(example_, example_t) 39 * In example.c: 40 * malloc_tsd_data(, example_, example_t, EX_INITIALIZER) 41 * malloc_tsd_funcs(, example_, example_t, EX_INITIALIZER, 42 * example_tsd_cleanup) 43 * 44 * The result is a set of generated functions, e.g.: 45 * 46 * bool example_tsd_boot(void) {...} 47 * example_t *example_tsd_get() {...} 48 * void example_tsd_set(example_t *val) {...} 49 * 50 * Note that all of the functions deal in terms of (a_type *) rather than 51 * (a_type) so that it is possible to support non-pointer types (unlike 52 * pthreads TSD). example_tsd_cleanup() is passed an (a_type *) pointer that is 53 * cast to (void *). This means that the cleanup function needs to cast the 54 * function argument to (a_type *), then dereference the resulting pointer to 55 * access fields, e.g. 56 * 57 * void 58 * example_tsd_cleanup(void *arg) 59 * { 60 * example_t *example = (example_t *)arg; 61 * 62 * example->x = 42; 63 * [...] 64 * if ([want the cleanup function to be called again]) 65 * example_tsd_set(example); 66 * } 67 * 68 * If example_tsd_set() is called within example_tsd_cleanup(), it will be 69 * called again. This is similar to how pthreads TSD destruction works, except 70 * that pthreads only calls the cleanup function again if the value was set to 71 * non-NULL. 72 */ 73 74 /* malloc_tsd_types(). */ 75 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 76 #define malloc_tsd_types(a_name, a_type) 77 #elif (defined(JEMALLOC_TLS)) 78 #define malloc_tsd_types(a_name, a_type) 79 #elif (defined(_WIN32)) 80 #define malloc_tsd_types(a_name, a_type) \ 81 typedef struct { \ 82 bool initialized; \ 83 a_type val; \ 84 } a_name##tsd_wrapper_t; 85 #else 86 #define malloc_tsd_types(a_name, a_type) \ 87 typedef struct { \ 88 bool initialized; \ 89 a_type val; \ 90 } a_name##tsd_wrapper_t; 91 #endif 92 93 /* malloc_tsd_protos(). */ 94 #define malloc_tsd_protos(a_attr, a_name, a_type) \ 95 a_attr bool \ 96 a_name##tsd_boot0(void); \ 97 a_attr void \ 98 a_name##tsd_boot1(void); \ 99 a_attr bool \ 100 a_name##tsd_boot(void); \ 101 a_attr a_type * \ 102 a_name##tsd_get(void); \ 103 a_attr void \ 104 a_name##tsd_set(a_type *val); 105 106 /* malloc_tsd_externs(). */ 107 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 108 #define malloc_tsd_externs(a_name, a_type) \ 109 extern __thread a_type a_name##tsd_tls; \ 110 extern __thread bool a_name##tsd_initialized; \ 111 extern bool a_name##tsd_booted; 112 #elif (defined(JEMALLOC_TLS)) 113 #define malloc_tsd_externs(a_name, a_type) \ 114 extern __thread a_type a_name##tsd_tls; \ 115 extern pthread_key_t a_name##tsd_tsd; \ 116 extern bool a_name##tsd_booted; 117 #elif (defined(_WIN32)) 118 #define malloc_tsd_externs(a_name, a_type) \ 119 extern DWORD a_name##tsd_tsd; \ 120 extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \ 121 extern bool a_name##tsd_booted; 122 #else 123 #define malloc_tsd_externs(a_name, a_type) \ 124 extern pthread_key_t a_name##tsd_tsd; \ 125 extern tsd_init_head_t a_name##tsd_init_head; \ 126 extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \ 127 extern bool a_name##tsd_booted; 128 #endif 129 130 /* malloc_tsd_data(). */ 131 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 132 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 133 a_attr __thread a_type JEMALLOC_TLS_MODEL \ 134 a_name##tsd_tls = a_initializer; \ 135 a_attr __thread bool JEMALLOC_TLS_MODEL \ 136 a_name##tsd_initialized = false; \ 137 a_attr bool a_name##tsd_booted = false; 138 #elif (defined(JEMALLOC_TLS)) 139 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 140 a_attr __thread a_type JEMALLOC_TLS_MODEL \ 141 a_name##tsd_tls = a_initializer; \ 142 a_attr pthread_key_t a_name##tsd_tsd; \ 143 a_attr bool a_name##tsd_booted = false; 144 #elif (defined(_WIN32)) 145 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 146 a_attr DWORD a_name##tsd_tsd; \ 147 a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \ 148 false, \ 149 a_initializer \ 150 }; \ 151 a_attr bool a_name##tsd_booted = false; 152 #else 153 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 154 a_attr pthread_key_t a_name##tsd_tsd; \ 155 a_attr tsd_init_head_t a_name##tsd_init_head = { \ 156 ql_head_initializer(blocks), \ 157 MALLOC_MUTEX_INITIALIZER \ 158 }; \ 159 a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \ 160 false, \ 161 a_initializer \ 162 }; \ 163 a_attr bool a_name##tsd_booted = false; 164 #endif 165 166 /* malloc_tsd_funcs(). */ 167 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 168 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 169 a_cleanup) \ 170 /* Initialization/cleanup. */ \ 171 a_attr bool \ 172 a_name##tsd_cleanup_wrapper(void) \ 173 { \ 174 \ 175 if (a_name##tsd_initialized) { \ 176 a_name##tsd_initialized = false; \ 177 a_cleanup(&a_name##tsd_tls); \ 178 } \ 179 return (a_name##tsd_initialized); \ 180 } \ 181 a_attr bool \ 182 a_name##tsd_boot0(void) \ 183 { \ 184 \ 185 if (a_cleanup != malloc_tsd_no_cleanup) { \ 186 malloc_tsd_cleanup_register( \ 187 &a_name##tsd_cleanup_wrapper); \ 188 } \ 189 a_name##tsd_booted = true; \ 190 return (false); \ 191 } \ 192 a_attr void \ 193 a_name##tsd_boot1(void) \ 194 { \ 195 \ 196 /* Do nothing. */ \ 197 } \ 198 a_attr bool \ 199 a_name##tsd_boot(void) \ 200 { \ 201 \ 202 return (a_name##tsd_boot0()); \ 203 } \ 204 /* Get/set. */ \ 205 a_attr a_type * \ 206 a_name##tsd_get(void) \ 207 { \ 208 \ 209 assert(a_name##tsd_booted); \ 210 return (&a_name##tsd_tls); \ 211 } \ 212 a_attr void \ 213 a_name##tsd_set(a_type *val) \ 214 { \ 215 \ 216 assert(a_name##tsd_booted); \ 217 a_name##tsd_tls = (*val); \ 218 if (a_cleanup != malloc_tsd_no_cleanup) \ 219 a_name##tsd_initialized = true; \ 220 } 221 #elif (defined(JEMALLOC_TLS)) 222 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 223 a_cleanup) \ 224 /* Initialization/cleanup. */ \ 225 a_attr bool \ 226 a_name##tsd_boot0(void) \ 227 { \ 228 \ 229 if (a_cleanup != malloc_tsd_no_cleanup) { \ 230 if (pthread_key_create(&a_name##tsd_tsd, a_cleanup) != \ 231 0) \ 232 return (true); \ 233 } \ 234 a_name##tsd_booted = true; \ 235 return (false); \ 236 } \ 237 a_attr void \ 238 a_name##tsd_boot1(void) \ 239 { \ 240 \ 241 /* Do nothing. */ \ 242 } \ 243 a_attr bool \ 244 a_name##tsd_boot(void) \ 245 { \ 246 \ 247 return (a_name##tsd_boot0()); \ 248 } \ 249 /* Get/set. */ \ 250 a_attr a_type * \ 251 a_name##tsd_get(void) \ 252 { \ 253 \ 254 assert(a_name##tsd_booted); \ 255 return (&a_name##tsd_tls); \ 256 } \ 257 a_attr void \ 258 a_name##tsd_set(a_type *val) \ 259 { \ 260 \ 261 assert(a_name##tsd_booted); \ 262 a_name##tsd_tls = (*val); \ 263 if (a_cleanup != malloc_tsd_no_cleanup) { \ 264 if (pthread_setspecific(a_name##tsd_tsd, \ 265 (void *)(&a_name##tsd_tls))) { \ 266 malloc_write("<jemalloc>: Error" \ 267 " setting TSD for "#a_name"\n"); \ 268 if (opt_abort) \ 269 abort(); \ 270 } \ 271 } \ 272 } 273 #elif (defined(_WIN32)) 274 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 275 a_cleanup) \ 276 /* Initialization/cleanup. */ \ 277 a_attr bool \ 278 a_name##tsd_cleanup_wrapper(void) \ 279 { \ 280 DWORD error = GetLastError(); \ 281 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ 282 TlsGetValue(a_name##tsd_tsd); \ 283 SetLastError(error); \ 284 \ 285 if (wrapper == NULL) \ 286 return (false); \ 287 if (a_cleanup != malloc_tsd_no_cleanup && \ 288 wrapper->initialized) { \ 289 wrapper->initialized = false; \ 290 a_cleanup(&wrapper->val); \ 291 if (wrapper->initialized) { \ 292 /* Trigger another cleanup round. */ \ 293 return (true); \ 294 } \ 295 } \ 296 malloc_tsd_dalloc(wrapper); \ 297 return (false); \ 298 } \ 299 a_attr void \ 300 a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \ 301 { \ 302 \ 303 if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) { \ 304 malloc_write("<jemalloc>: Error setting" \ 305 " TSD for "#a_name"\n"); \ 306 abort(); \ 307 } \ 308 } \ 309 a_attr a_name##tsd_wrapper_t * \ 310 a_name##tsd_wrapper_get(void) \ 311 { \ 312 DWORD error = GetLastError(); \ 313 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ 314 TlsGetValue(a_name##tsd_tsd); \ 315 SetLastError(error); \ 316 \ 317 if (unlikely(wrapper == NULL)) { \ 318 wrapper = (a_name##tsd_wrapper_t *) \ 319 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 320 if (wrapper == NULL) { \ 321 malloc_write("<jemalloc>: Error allocating" \ 322 " TSD for "#a_name"\n"); \ 323 abort(); \ 324 } else { \ 325 wrapper->initialized = false; \ 326 wrapper->val = a_initializer; \ 327 } \ 328 a_name##tsd_wrapper_set(wrapper); \ 329 } \ 330 return (wrapper); \ 331 } \ 332 a_attr bool \ 333 a_name##tsd_boot0(void) \ 334 { \ 335 \ 336 a_name##tsd_tsd = TlsAlloc(); \ 337 if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES) \ 338 return (true); \ 339 if (a_cleanup != malloc_tsd_no_cleanup) { \ 340 malloc_tsd_cleanup_register( \ 341 &a_name##tsd_cleanup_wrapper); \ 342 } \ 343 a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \ 344 a_name##tsd_booted = true; \ 345 return (false); \ 346 } \ 347 a_attr void \ 348 a_name##tsd_boot1(void) \ 349 { \ 350 a_name##tsd_wrapper_t *wrapper; \ 351 wrapper = (a_name##tsd_wrapper_t *) \ 352 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 353 if (wrapper == NULL) { \ 354 malloc_write("<jemalloc>: Error allocating" \ 355 " TSD for "#a_name"\n"); \ 356 abort(); \ 357 } \ 358 memcpy(wrapper, &a_name##tsd_boot_wrapper, \ 359 sizeof(a_name##tsd_wrapper_t)); \ 360 a_name##tsd_wrapper_set(wrapper); \ 361 } \ 362 a_attr bool \ 363 a_name##tsd_boot(void) \ 364 { \ 365 \ 366 if (a_name##tsd_boot0()) \ 367 return (true); \ 368 a_name##tsd_boot1(); \ 369 return (false); \ 370 } \ 371 /* Get/set. */ \ 372 a_attr a_type * \ 373 a_name##tsd_get(void) \ 374 { \ 375 a_name##tsd_wrapper_t *wrapper; \ 376 \ 377 assert(a_name##tsd_booted); \ 378 wrapper = a_name##tsd_wrapper_get(); \ 379 return (&wrapper->val); \ 380 } \ 381 a_attr void \ 382 a_name##tsd_set(a_type *val) \ 383 { \ 384 a_name##tsd_wrapper_t *wrapper; \ 385 \ 386 assert(a_name##tsd_booted); \ 387 wrapper = a_name##tsd_wrapper_get(); \ 388 wrapper->val = *(val); \ 389 if (a_cleanup != malloc_tsd_no_cleanup) \ 390 wrapper->initialized = true; \ 391 } 392 #else 393 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 394 a_cleanup) \ 395 /* Initialization/cleanup. */ \ 396 a_attr void \ 397 a_name##tsd_cleanup_wrapper(void *arg) \ 398 { \ 399 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)arg; \ 400 \ 401 if (a_cleanup != malloc_tsd_no_cleanup && \ 402 wrapper->initialized) { \ 403 wrapper->initialized = false; \ 404 a_cleanup(&wrapper->val); \ 405 if (wrapper->initialized) { \ 406 /* Trigger another cleanup round. */ \ 407 if (pthread_setspecific(a_name##tsd_tsd, \ 408 (void *)wrapper)) { \ 409 malloc_write("<jemalloc>: Error" \ 410 " setting TSD for "#a_name"\n"); \ 411 if (opt_abort) \ 412 abort(); \ 413 } \ 414 return; \ 415 } \ 416 } \ 417 malloc_tsd_dalloc(wrapper); \ 418 } \ 419 a_attr void \ 420 a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \ 421 { \ 422 \ 423 if (pthread_setspecific(a_name##tsd_tsd, \ 424 (void *)wrapper)) { \ 425 malloc_write("<jemalloc>: Error setting" \ 426 " TSD for "#a_name"\n"); \ 427 abort(); \ 428 } \ 429 } \ 430 a_attr a_name##tsd_wrapper_t * \ 431 a_name##tsd_wrapper_get(void) \ 432 { \ 433 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ 434 pthread_getspecific(a_name##tsd_tsd); \ 435 \ 436 if (unlikely(wrapper == NULL)) { \ 437 tsd_init_block_t block; \ 438 wrapper = tsd_init_check_recursion( \ 439 &a_name##tsd_init_head, &block); \ 440 if (wrapper) \ 441 return (wrapper); \ 442 wrapper = (a_name##tsd_wrapper_t *) \ 443 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 444 block.data = wrapper; \ 445 if (wrapper == NULL) { \ 446 malloc_write("<jemalloc>: Error allocating" \ 447 " TSD for "#a_name"\n"); \ 448 abort(); \ 449 } else { \ 450 wrapper->initialized = false; \ 451 wrapper->val = a_initializer; \ 452 } \ 453 a_name##tsd_wrapper_set(wrapper); \ 454 tsd_init_finish(&a_name##tsd_init_head, &block); \ 455 } \ 456 return (wrapper); \ 457 } \ 458 a_attr bool \ 459 a_name##tsd_boot0(void) \ 460 { \ 461 \ 462 if (pthread_key_create(&a_name##tsd_tsd, \ 463 a_name##tsd_cleanup_wrapper) != 0) \ 464 return (true); \ 465 a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \ 466 a_name##tsd_booted = true; \ 467 return (false); \ 468 } \ 469 a_attr void \ 470 a_name##tsd_boot1(void) \ 471 { \ 472 a_name##tsd_wrapper_t *wrapper; \ 473 wrapper = (a_name##tsd_wrapper_t *) \ 474 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 475 if (wrapper == NULL) { \ 476 malloc_write("<jemalloc>: Error allocating" \ 477 " TSD for "#a_name"\n"); \ 478 abort(); \ 479 } \ 480 memcpy(wrapper, &a_name##tsd_boot_wrapper, \ 481 sizeof(a_name##tsd_wrapper_t)); \ 482 a_name##tsd_wrapper_set(wrapper); \ 483 } \ 484 a_attr bool \ 485 a_name##tsd_boot(void) \ 486 { \ 487 \ 488 if (a_name##tsd_boot0()) \ 489 return (true); \ 490 a_name##tsd_boot1(); \ 491 return (false); \ 492 } \ 493 /* Get/set. */ \ 494 a_attr a_type * \ 495 a_name##tsd_get(void) \ 496 { \ 497 a_name##tsd_wrapper_t *wrapper; \ 498 \ 499 assert(a_name##tsd_booted); \ 500 wrapper = a_name##tsd_wrapper_get(); \ 501 return (&wrapper->val); \ 502 } \ 503 a_attr void \ 504 a_name##tsd_set(a_type *val) \ 505 { \ 506 a_name##tsd_wrapper_t *wrapper; \ 507 \ 508 assert(a_name##tsd_booted); \ 509 wrapper = a_name##tsd_wrapper_get(); \ 510 wrapper->val = *(val); \ 511 if (a_cleanup != malloc_tsd_no_cleanup) \ 512 wrapper->initialized = true; \ 513 } 514 #endif 515 516 #endif /* JEMALLOC_H_TYPES */ 517 /******************************************************************************/ 518 #ifdef JEMALLOC_H_STRUCTS 519 520 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 521 !defined(_WIN32)) 522 struct tsd_init_block_s { 523 ql_elm(tsd_init_block_t) link; 524 pthread_t thread; 525 void *data; 526 }; 527 struct tsd_init_head_s { 528 ql_head(tsd_init_block_t) blocks; 529 malloc_mutex_t lock; 530 }; 531 #endif 532 533 #define MALLOC_TSD \ 534 /* O(name, type) */ \ 535 O(tcache, tcache_t *) \ 536 O(thread_allocated, uint64_t) \ 537 O(thread_deallocated, uint64_t) \ 538 O(prof_tdata, prof_tdata_t *) \ 539 O(arena, arena_t *) \ 540 O(arenas_tdata, arena_tdata_t *) \ 541 O(narenas_tdata, unsigned) \ 542 O(arenas_tdata_bypass, bool) \ 543 O(tcache_enabled, tcache_enabled_t) \ 544 O(quarantine, quarantine_t *) \ 545 546 #define TSD_INITIALIZER { \ 547 tsd_state_uninitialized, \ 548 NULL, \ 549 0, \ 550 0, \ 551 NULL, \ 552 NULL, \ 553 NULL, \ 554 0, \ 555 false, \ 556 tcache_enabled_default, \ 557 NULL \ 558 } 559 560 struct tsd_s { 561 tsd_state_t state; 562 #define O(n, t) \ 563 t n; 564 MALLOC_TSD 565 #undef O 566 }; 567 568 static const tsd_t tsd_initializer = TSD_INITIALIZER; 569 570 malloc_tsd_types(, tsd_t) 571 572 #endif /* JEMALLOC_H_STRUCTS */ 573 /******************************************************************************/ 574 #ifdef JEMALLOC_H_EXTERNS 575 576 void *malloc_tsd_malloc(size_t size); 577 void malloc_tsd_dalloc(void *wrapper); 578 void malloc_tsd_no_cleanup(void *arg); 579 void malloc_tsd_cleanup_register(bool (*f)(void)); 580 bool malloc_tsd_boot0(void); 581 void malloc_tsd_boot1(void); 582 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 583 !defined(_WIN32)) 584 void *tsd_init_check_recursion(tsd_init_head_t *head, 585 tsd_init_block_t *block); 586 void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block); 587 #endif 588 void tsd_cleanup(void *arg); 589 590 #endif /* JEMALLOC_H_EXTERNS */ 591 /******************************************************************************/ 592 #ifdef JEMALLOC_H_INLINES 593 594 #ifndef JEMALLOC_ENABLE_INLINE 595 malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t) 596 597 tsd_t *tsd_fetch(void); 598 bool tsd_nominal(tsd_t *tsd); 599 #define O(n, t) \ 600 t *tsd_##n##p_get(tsd_t *tsd); \ 601 t tsd_##n##_get(tsd_t *tsd); \ 602 void tsd_##n##_set(tsd_t *tsd, t n); 603 MALLOC_TSD 604 #undef O 605 #endif 606 607 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_)) 608 malloc_tsd_externs(, tsd_t) 609 malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup) 610 611 JEMALLOC_ALWAYS_INLINE tsd_t * 612 tsd_fetch(void) 613 { 614 tsd_t *tsd = tsd_get(); 615 616 if (unlikely(tsd->state != tsd_state_nominal)) { 617 if (tsd->state == tsd_state_uninitialized) { 618 tsd->state = tsd_state_nominal; 619 /* Trigger cleanup handler registration. */ 620 tsd_set(tsd); 621 } else if (tsd->state == tsd_state_purgatory) { 622 tsd->state = tsd_state_reincarnated; 623 tsd_set(tsd); 624 } else 625 assert(tsd->state == tsd_state_reincarnated); 626 } 627 628 return (tsd); 629 } 630 631 JEMALLOC_INLINE bool 632 tsd_nominal(tsd_t *tsd) 633 { 634 635 return (tsd->state == tsd_state_nominal); 636 } 637 638 #define O(n, t) \ 639 JEMALLOC_ALWAYS_INLINE t * \ 640 tsd_##n##p_get(tsd_t *tsd) \ 641 { \ 642 \ 643 return (&tsd->n); \ 644 } \ 645 \ 646 JEMALLOC_ALWAYS_INLINE t \ 647 tsd_##n##_get(tsd_t *tsd) \ 648 { \ 649 \ 650 return (*tsd_##n##p_get(tsd)); \ 651 } \ 652 \ 653 JEMALLOC_ALWAYS_INLINE void \ 654 tsd_##n##_set(tsd_t *tsd, t n) \ 655 { \ 656 \ 657 assert(tsd->state == tsd_state_nominal); \ 658 tsd->n = n; \ 659 } 660 MALLOC_TSD 661 #undef O 662 #endif 663 664 #endif /* JEMALLOC_H_INLINES */ 665 /******************************************************************************/ 666
