1 /* 2 * Copyright (C) 2008 The Android Open Source Project 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * * Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * * Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in 12 * the documentation and/or other materials provided with the 13 * distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 18 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 19 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 22 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 25 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 #include <sys/types.h> 29 #include <unistd.h> 30 #include <signal.h> 31 #include <stdint.h> 32 #include <stdio.h> 33 #include <stdlib.h> 34 #include <errno.h> 35 #include <sys/atomics.h> 36 #include <bionic_tls.h> 37 #include <sys/mman.h> 38 #include <pthread.h> 39 #include <time.h> 40 #include "pthread_internal.h" 41 #include "thread_private.h" 42 #include <limits.h> 43 #include <memory.h> 44 #include <assert.h> 45 #include <malloc.h> 46 #include <bionic_futex.h> 47 #include <bionic_atomic_inline.h> 48 #include <sys/prctl.h> 49 #include <sys/stat.h> 50 #include <fcntl.h> 51 #include <stdio.h> 52 #include <bionic_pthread.h> 53 54 extern void pthread_debug_mutex_lock_check(pthread_mutex_t *mutex); 55 extern void pthread_debug_mutex_unlock_check(pthread_mutex_t *mutex); 56 57 extern int __pthread_clone(int (*fn)(void*), void *child_stack, int flags, void *arg); 58 extern void _exit_with_stack_teardown(void * stackBase, int stackSize, int retCode); 59 extern void _exit_thread(int retCode); 60 extern int __set_errno(int); 61 62 int __futex_wake_ex(volatile void *ftx, int pshared, int val) 63 { 64 return __futex_syscall3(ftx, pshared ? FUTEX_WAKE : FUTEX_WAKE_PRIVATE, val); 65 } 66 67 int __futex_wait_ex(volatile void *ftx, int pshared, int val, const struct timespec *timeout) 68 { 69 return __futex_syscall4(ftx, pshared ? FUTEX_WAIT : FUTEX_WAIT_PRIVATE, val, timeout); 70 } 71 72 #define __likely(cond) __builtin_expect(!!(cond), 1) 73 #define __unlikely(cond) __builtin_expect(!!(cond), 0) 74 75 #ifdef __i386__ 76 #define ATTRIBUTES __attribute__((noinline)) __attribute__((fastcall)) 77 #else 78 #define ATTRIBUTES __attribute__((noinline)) 79 #endif 80 81 void ATTRIBUTES _thread_created_hook(pid_t thread_id); 82 83 #define PTHREAD_ATTR_FLAG_DETACHED 0x00000001 84 #define PTHREAD_ATTR_FLAG_USER_STACK 0x00000002 85 86 #define DEFAULT_STACKSIZE (1024 * 1024) 87 88 static pthread_mutex_t mmap_lock = PTHREAD_MUTEX_INITIALIZER; 89 90 91 static const pthread_attr_t gDefaultPthreadAttr = { 92 .flags = 0, 93 .stack_base = NULL, 94 .stack_size = DEFAULT_STACKSIZE, 95 .guard_size = PAGE_SIZE, 96 .sched_policy = SCHED_NORMAL, 97 .sched_priority = 0 98 }; 99 100 #define INIT_THREADS 1 101 102 static pthread_internal_t* gThreadList = NULL; 103 static pthread_mutex_t gThreadListLock = PTHREAD_MUTEX_INITIALIZER; 104 static pthread_mutex_t gDebuggerNotificationLock = PTHREAD_MUTEX_INITIALIZER; 105 106 107 /* we simply malloc/free the internal pthread_internal_t structures. we may 108 * want to use a different allocation scheme in the future, but this one should 109 * be largely enough 110 */ 111 static pthread_internal_t* 112 _pthread_internal_alloc(void) 113 { 114 pthread_internal_t* thread; 115 116 thread = calloc( sizeof(*thread), 1 ); 117 if (thread) 118 thread->intern = 1; 119 120 return thread; 121 } 122 123 static void 124 _pthread_internal_free( pthread_internal_t* thread ) 125 { 126 if (thread && thread->intern) { 127 thread->intern = 0; /* just in case */ 128 free (thread); 129 } 130 } 131 132 133 static void 134 _pthread_internal_remove_locked( pthread_internal_t* thread ) 135 { 136 thread->next->pref = thread->pref; 137 thread->pref[0] = thread->next; 138 } 139 140 static void 141 _pthread_internal_remove( pthread_internal_t* thread ) 142 { 143 pthread_mutex_lock(&gThreadListLock); 144 _pthread_internal_remove_locked(thread); 145 pthread_mutex_unlock(&gThreadListLock); 146 } 147 148 __LIBC_ABI_PRIVATE__ void 149 _pthread_internal_add( pthread_internal_t* thread ) 150 { 151 pthread_mutex_lock(&gThreadListLock); 152 thread->pref = &gThreadList; 153 thread->next = thread->pref[0]; 154 if (thread->next) 155 thread->next->pref = &thread->next; 156 thread->pref[0] = thread; 157 pthread_mutex_unlock(&gThreadListLock); 158 } 159 160 __LIBC_ABI_PRIVATE__ pthread_internal_t* 161 __get_thread(void) 162 { 163 void** tls = (void**)__get_tls(); 164 165 return (pthread_internal_t*) tls[TLS_SLOT_THREAD_ID]; 166 } 167 168 169 void* 170 __get_stack_base(int *p_stack_size) 171 { 172 pthread_internal_t* thread = __get_thread(); 173 174 *p_stack_size = thread->attr.stack_size; 175 return thread->attr.stack_base; 176 } 177 178 179 void __init_tls(void** tls, void* thread) 180 { 181 int nn; 182 183 ((pthread_internal_t*)thread)->tls = tls; 184 185 // slot 0 must point to the tls area, this is required by the implementation 186 // of the x86 Linux kernel thread-local-storage 187 tls[TLS_SLOT_SELF] = (void*)tls; 188 tls[TLS_SLOT_THREAD_ID] = thread; 189 for (nn = TLS_SLOT_ERRNO; nn < BIONIC_TLS_SLOTS; nn++) 190 tls[nn] = 0; 191 192 __set_tls( (void*)tls ); 193 } 194 195 196 /* 197 * This trampoline is called from the assembly clone() function 198 */ 199 void __thread_entry(int (*func)(void*), void *arg, void **tls) 200 { 201 int retValue; 202 pthread_internal_t * thrInfo; 203 204 // Wait for our creating thread to release us. This lets it have time to 205 // notify gdb about this thread before it starts doing anything. 206 // 207 // This also provides the memory barrier needed to ensure that all memory 208 // accesses previously made by the creating thread are visible to us. 209 pthread_mutex_t * start_mutex = (pthread_mutex_t *)&tls[TLS_SLOT_SELF]; 210 pthread_mutex_lock(start_mutex); 211 pthread_mutex_destroy(start_mutex); 212 213 thrInfo = (pthread_internal_t *) tls[TLS_SLOT_THREAD_ID]; 214 215 __init_tls( tls, thrInfo ); 216 217 pthread_exit( (void*)func(arg) ); 218 } 219 220 __LIBC_ABI_PRIVATE__ 221 void _init_thread(pthread_internal_t * thread, pid_t kernel_id, pthread_attr_t * attr, void * stack_base) 222 { 223 if (attr == NULL) { 224 thread->attr = gDefaultPthreadAttr; 225 } else { 226 thread->attr = *attr; 227 } 228 thread->attr.stack_base = stack_base; 229 thread->kernel_id = kernel_id; 230 231 // set the scheduling policy/priority of the thread 232 if (thread->attr.sched_policy != SCHED_NORMAL) { 233 struct sched_param param; 234 param.sched_priority = thread->attr.sched_priority; 235 sched_setscheduler(kernel_id, thread->attr.sched_policy, ¶m); 236 } 237 238 pthread_cond_init(&thread->join_cond, NULL); 239 thread->join_count = 0; 240 241 thread->cleanup_stack = NULL; 242 } 243 244 245 /* XXX stacks not reclaimed if thread spawn fails */ 246 /* XXX stacks address spaces should be reused if available again */ 247 248 static void *mkstack(size_t size, size_t guard_size) 249 { 250 void * stack; 251 252 pthread_mutex_lock(&mmap_lock); 253 254 stack = mmap(NULL, size, 255 PROT_READ | PROT_WRITE, 256 MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, 257 -1, 0); 258 259 if(stack == MAP_FAILED) { 260 stack = NULL; 261 goto done; 262 } 263 264 if(mprotect(stack, guard_size, PROT_NONE)){ 265 munmap(stack, size); 266 stack = NULL; 267 goto done; 268 } 269 270 done: 271 pthread_mutex_unlock(&mmap_lock); 272 return stack; 273 } 274 275 /* 276 * Create a new thread. The thread's stack is laid out like so: 277 * 278 * +---------------------------+ 279 * | pthread_internal_t | 280 * +---------------------------+ 281 * | | 282 * | TLS area | 283 * | | 284 * +---------------------------+ 285 * | | 286 * . . 287 * . stack area . 288 * . . 289 * | | 290 * +---------------------------+ 291 * | guard page | 292 * +---------------------------+ 293 * 294 * note that TLS[0] must be a pointer to itself, this is required 295 * by the thread-local storage implementation of the x86 Linux 296 * kernel, where the TLS pointer is read by reading fs:[0] 297 */ 298 int pthread_create(pthread_t *thread_out, pthread_attr_t const * attr, 299 void *(*start_routine)(void *), void * arg) 300 { 301 char* stack; 302 void** tls; 303 int tid; 304 pthread_mutex_t * start_mutex; 305 pthread_internal_t * thread; 306 int madestack = 0; 307 int old_errno = errno; 308 309 /* this will inform the rest of the C library that at least one thread 310 * was created. this will enforce certain functions to acquire/release 311 * locks (e.g. atexit()) to protect shared global structures. 312 * 313 * this works because pthread_create() is not called by the C library 314 * initialization routine that sets up the main thread's data structures. 315 */ 316 __isthreaded = 1; 317 318 thread = _pthread_internal_alloc(); 319 if (thread == NULL) 320 return ENOMEM; 321 322 if (attr == NULL) { 323 attr = &gDefaultPthreadAttr; 324 } 325 326 // make sure the stack is PAGE_SIZE aligned 327 size_t stackSize = (attr->stack_size + 328 (PAGE_SIZE-1)) & ~(PAGE_SIZE-1); 329 330 if (!attr->stack_base) { 331 stack = mkstack(stackSize, attr->guard_size); 332 if(stack == NULL) { 333 _pthread_internal_free(thread); 334 return ENOMEM; 335 } 336 madestack = 1; 337 } else { 338 stack = attr->stack_base; 339 } 340 341 // Make room for TLS 342 tls = (void**)(stack + stackSize - BIONIC_TLS_SLOTS*sizeof(void*)); 343 344 // Create a mutex for the thread in TLS_SLOT_SELF to wait on once it starts so we can keep 345 // it from doing anything until after we notify the debugger about it 346 // 347 // This also provides the memory barrier we need to ensure that all 348 // memory accesses previously performed by this thread are visible to 349 // the new thread. 350 start_mutex = (pthread_mutex_t *) &tls[TLS_SLOT_SELF]; 351 pthread_mutex_init(start_mutex, NULL); 352 pthread_mutex_lock(start_mutex); 353 354 tls[TLS_SLOT_THREAD_ID] = thread; 355 356 tid = __pthread_clone((int(*)(void*))start_routine, tls, 357 CLONE_FILES | CLONE_FS | CLONE_VM | CLONE_SIGHAND 358 | CLONE_THREAD | CLONE_SYSVSEM | CLONE_DETACHED, 359 arg); 360 361 if(tid < 0) { 362 int result; 363 if (madestack) 364 munmap(stack, stackSize); 365 _pthread_internal_free(thread); 366 result = errno; 367 errno = old_errno; 368 return result; 369 } 370 371 _init_thread(thread, tid, (pthread_attr_t*)attr, stack); 372 373 _pthread_internal_add(thread); 374 375 if (!madestack) 376 thread->attr.flags |= PTHREAD_ATTR_FLAG_USER_STACK; 377 378 // Notify any debuggers about the new thread 379 pthread_mutex_lock(&gDebuggerNotificationLock); 380 _thread_created_hook(tid); 381 pthread_mutex_unlock(&gDebuggerNotificationLock); 382 383 // Let the thread do it's thing 384 pthread_mutex_unlock(start_mutex); 385 386 *thread_out = (pthread_t)thread; 387 return 0; 388 } 389 390 391 int pthread_attr_init(pthread_attr_t * attr) 392 { 393 *attr = gDefaultPthreadAttr; 394 return 0; 395 } 396 397 int pthread_attr_destroy(pthread_attr_t * attr) 398 { 399 memset(attr, 0x42, sizeof(pthread_attr_t)); 400 return 0; 401 } 402 403 int pthread_attr_setdetachstate(pthread_attr_t * attr, int state) 404 { 405 if (state == PTHREAD_CREATE_DETACHED) { 406 attr->flags |= PTHREAD_ATTR_FLAG_DETACHED; 407 } else if (state == PTHREAD_CREATE_JOINABLE) { 408 attr->flags &= ~PTHREAD_ATTR_FLAG_DETACHED; 409 } else { 410 return EINVAL; 411 } 412 return 0; 413 } 414 415 int pthread_attr_getdetachstate(pthread_attr_t const * attr, int * state) 416 { 417 *state = (attr->flags & PTHREAD_ATTR_FLAG_DETACHED) 418 ? PTHREAD_CREATE_DETACHED 419 : PTHREAD_CREATE_JOINABLE; 420 return 0; 421 } 422 423 int pthread_attr_setschedpolicy(pthread_attr_t * attr, int policy) 424 { 425 attr->sched_policy = policy; 426 return 0; 427 } 428 429 int pthread_attr_getschedpolicy(pthread_attr_t const * attr, int * policy) 430 { 431 *policy = attr->sched_policy; 432 return 0; 433 } 434 435 int pthread_attr_setschedparam(pthread_attr_t * attr, struct sched_param const * param) 436 { 437 attr->sched_priority = param->sched_priority; 438 return 0; 439 } 440 441 int pthread_attr_getschedparam(pthread_attr_t const * attr, struct sched_param * param) 442 { 443 param->sched_priority = attr->sched_priority; 444 return 0; 445 } 446 447 int pthread_attr_setstacksize(pthread_attr_t * attr, size_t stack_size) 448 { 449 if ((stack_size & (PAGE_SIZE - 1) || stack_size < PTHREAD_STACK_MIN)) { 450 return EINVAL; 451 } 452 attr->stack_size = stack_size; 453 return 0; 454 } 455 456 int pthread_attr_getstacksize(pthread_attr_t const * attr, size_t * stack_size) 457 { 458 *stack_size = attr->stack_size; 459 return 0; 460 } 461 462 int pthread_attr_setstackaddr(pthread_attr_t * attr, void * stack_addr) 463 { 464 #if 1 465 // It's not clear if this is setting the top or bottom of the stack, so don't handle it for now. 466 return ENOSYS; 467 #else 468 if ((uint32_t)stack_addr & (PAGE_SIZE - 1)) { 469 return EINVAL; 470 } 471 attr->stack_base = stack_addr; 472 return 0; 473 #endif 474 } 475 476 int pthread_attr_getstackaddr(pthread_attr_t const * attr, void ** stack_addr) 477 { 478 *stack_addr = (char*)attr->stack_base + attr->stack_size; 479 return 0; 480 } 481 482 int pthread_attr_setstack(pthread_attr_t * attr, void * stack_base, size_t stack_size) 483 { 484 if ((stack_size & (PAGE_SIZE - 1) || stack_size < PTHREAD_STACK_MIN)) { 485 return EINVAL; 486 } 487 if ((uint32_t)stack_base & (PAGE_SIZE - 1)) { 488 return EINVAL; 489 } 490 attr->stack_base = stack_base; 491 attr->stack_size = stack_size; 492 return 0; 493 } 494 495 int pthread_attr_getstack(pthread_attr_t const * attr, void ** stack_base, size_t * stack_size) 496 { 497 *stack_base = attr->stack_base; 498 *stack_size = attr->stack_size; 499 return 0; 500 } 501 502 int pthread_attr_setguardsize(pthread_attr_t * attr, size_t guard_size) 503 { 504 if (guard_size & (PAGE_SIZE - 1) || guard_size < PAGE_SIZE) { 505 return EINVAL; 506 } 507 508 attr->guard_size = guard_size; 509 return 0; 510 } 511 512 int pthread_attr_getguardsize(pthread_attr_t const * attr, size_t * guard_size) 513 { 514 *guard_size = attr->guard_size; 515 return 0; 516 } 517 518 int pthread_getattr_np(pthread_t thid, pthread_attr_t * attr) 519 { 520 pthread_internal_t * thread = (pthread_internal_t *)thid; 521 *attr = thread->attr; 522 return 0; 523 } 524 525 int pthread_attr_setscope(pthread_attr_t *attr, int scope) 526 { 527 if (scope == PTHREAD_SCOPE_SYSTEM) 528 return 0; 529 if (scope == PTHREAD_SCOPE_PROCESS) 530 return ENOTSUP; 531 532 return EINVAL; 533 } 534 535 int pthread_attr_getscope(pthread_attr_t const *attr) 536 { 537 return PTHREAD_SCOPE_SYSTEM; 538 } 539 540 541 /* CAVEAT: our implementation of pthread_cleanup_push/pop doesn't support C++ exceptions 542 * and thread cancelation 543 */ 544 545 void __pthread_cleanup_push( __pthread_cleanup_t* c, 546 __pthread_cleanup_func_t routine, 547 void* arg ) 548 { 549 pthread_internal_t* thread = __get_thread(); 550 551 c->__cleanup_routine = routine; 552 c->__cleanup_arg = arg; 553 c->__cleanup_prev = thread->cleanup_stack; 554 thread->cleanup_stack = c; 555 } 556 557 void __pthread_cleanup_pop( __pthread_cleanup_t* c, int execute ) 558 { 559 pthread_internal_t* thread = __get_thread(); 560 561 thread->cleanup_stack = c->__cleanup_prev; 562 if (execute) 563 c->__cleanup_routine(c->__cleanup_arg); 564 } 565 566 /* used by pthread_exit() to clean all TLS keys of the current thread */ 567 static void pthread_key_clean_all(void); 568 569 void pthread_exit(void * retval) 570 { 571 pthread_internal_t* thread = __get_thread(); 572 void* stack_base = thread->attr.stack_base; 573 int stack_size = thread->attr.stack_size; 574 int user_stack = (thread->attr.flags & PTHREAD_ATTR_FLAG_USER_STACK) != 0; 575 sigset_t mask; 576 577 // call the cleanup handlers first 578 while (thread->cleanup_stack) { 579 __pthread_cleanup_t* c = thread->cleanup_stack; 580 thread->cleanup_stack = c->__cleanup_prev; 581 c->__cleanup_routine(c->__cleanup_arg); 582 } 583 584 // call the TLS destructors, it is important to do that before removing this 585 // thread from the global list. this will ensure that if someone else deletes 586 // a TLS key, the corresponding value will be set to NULL in this thread's TLS 587 // space (see pthread_key_delete) 588 pthread_key_clean_all(); 589 590 // if the thread is detached, destroy the pthread_internal_t 591 // otherwise, keep it in memory and signal any joiners 592 if (thread->attr.flags & PTHREAD_ATTR_FLAG_DETACHED) { 593 _pthread_internal_remove(thread); 594 _pthread_internal_free(thread); 595 } else { 596 /* the join_count field is used to store the number of threads waiting for 597 * the termination of this thread with pthread_join(), 598 * 599 * if it is positive we need to signal the waiters, and we do not touch 600 * the count (it will be decremented by the waiters, the last one will 601 * also remove/free the thread structure 602 * 603 * if it is zero, we set the count value to -1 to indicate that the 604 * thread is in 'zombie' state: it has stopped executing, and its stack 605 * is gone (as well as its TLS area). when another thread calls pthread_join() 606 * on it, it will immediately free the thread and return. 607 */ 608 pthread_mutex_lock(&gThreadListLock); 609 thread->return_value = retval; 610 if (thread->join_count > 0) { 611 pthread_cond_broadcast(&thread->join_cond); 612 } else { 613 thread->join_count = -1; /* zombie thread */ 614 } 615 pthread_mutex_unlock(&gThreadListLock); 616 } 617 618 sigfillset(&mask); 619 sigdelset(&mask, SIGSEGV); 620 (void)sigprocmask(SIG_SETMASK, &mask, (sigset_t *)NULL); 621 622 // destroy the thread stack 623 if (user_stack) 624 _exit_thread((int)retval); 625 else 626 _exit_with_stack_teardown(stack_base, stack_size, (int)retval); 627 } 628 629 int pthread_join(pthread_t thid, void ** ret_val) 630 { 631 pthread_internal_t* thread = (pthread_internal_t*)thid; 632 int count; 633 634 // check that the thread still exists and is not detached 635 pthread_mutex_lock(&gThreadListLock); 636 637 for (thread = gThreadList; thread != NULL; thread = thread->next) 638 if (thread == (pthread_internal_t*)thid) 639 goto FoundIt; 640 641 pthread_mutex_unlock(&gThreadListLock); 642 return ESRCH; 643 644 FoundIt: 645 if (thread->attr.flags & PTHREAD_ATTR_FLAG_DETACHED) { 646 pthread_mutex_unlock(&gThreadListLock); 647 return EINVAL; 648 } 649 650 /* wait for thread death when needed 651 * 652 * if the 'join_count' is negative, this is a 'zombie' thread that 653 * is already dead and without stack/TLS 654 * 655 * otherwise, we need to increment 'join-count' and wait to be signaled 656 */ 657 count = thread->join_count; 658 if (count >= 0) { 659 thread->join_count += 1; 660 pthread_cond_wait( &thread->join_cond, &gThreadListLock ); 661 count = --thread->join_count; 662 } 663 if (ret_val) 664 *ret_val = thread->return_value; 665 666 /* remove thread descriptor when we're the last joiner or when the 667 * thread was already a zombie. 668 */ 669 if (count <= 0) { 670 _pthread_internal_remove_locked(thread); 671 _pthread_internal_free(thread); 672 } 673 pthread_mutex_unlock(&gThreadListLock); 674 return 0; 675 } 676 677 int pthread_detach( pthread_t thid ) 678 { 679 pthread_internal_t* thread; 680 int result = 0; 681 int flags; 682 683 pthread_mutex_lock(&gThreadListLock); 684 for (thread = gThreadList; thread != NULL; thread = thread->next) 685 if (thread == (pthread_internal_t*)thid) 686 goto FoundIt; 687 688 result = ESRCH; 689 goto Exit; 690 691 FoundIt: 692 do { 693 flags = thread->attr.flags; 694 695 if ( flags & PTHREAD_ATTR_FLAG_DETACHED ) { 696 /* thread is not joinable ! */ 697 result = EINVAL; 698 goto Exit; 699 } 700 } 701 while ( __bionic_cmpxchg( flags, flags | PTHREAD_ATTR_FLAG_DETACHED, 702 (volatile int*)&thread->attr.flags ) != 0 ); 703 Exit: 704 pthread_mutex_unlock(&gThreadListLock); 705 return result; 706 } 707 708 pthread_t pthread_self(void) 709 { 710 return (pthread_t)__get_thread(); 711 } 712 713 int pthread_equal(pthread_t one, pthread_t two) 714 { 715 return (one == two ? 1 : 0); 716 } 717 718 int pthread_getschedparam(pthread_t thid, int * policy, 719 struct sched_param * param) 720 { 721 int old_errno = errno; 722 723 pthread_internal_t * thread = (pthread_internal_t *)thid; 724 int err = sched_getparam(thread->kernel_id, param); 725 if (!err) { 726 *policy = sched_getscheduler(thread->kernel_id); 727 } else { 728 err = errno; 729 errno = old_errno; 730 } 731 return err; 732 } 733 734 int pthread_setschedparam(pthread_t thid, int policy, 735 struct sched_param const * param) 736 { 737 pthread_internal_t * thread = (pthread_internal_t *)thid; 738 int old_errno = errno; 739 int ret; 740 741 ret = sched_setscheduler(thread->kernel_id, policy, param); 742 if (ret < 0) { 743 ret = errno; 744 errno = old_errno; 745 } 746 return ret; 747 } 748 749 750 /* a mutex is implemented as a 32-bit integer holding the following fields 751 * 752 * bits: name description 753 * 31-16 tid owner thread's kernel id (recursive and errorcheck only) 754 * 15-14 type mutex type 755 * 13 shared process-shared flag 756 * 12-2 counter counter of recursive mutexes 757 * 1-0 state lock state (0, 1 or 2) 758 */ 759 760 /* Convenience macro, creates a mask of 'bits' bits that starts from 761 * the 'shift'-th least significant bit in a 32-bit word. 762 * 763 * Examples: FIELD_MASK(0,4) -> 0xf 764 * FIELD_MASK(16,9) -> 0x1ff0000 765 */ 766 #define FIELD_MASK(shift,bits) (((1 << (bits))-1) << (shift)) 767 768 /* This one is used to create a bit pattern from a given field value */ 769 #define FIELD_TO_BITS(val,shift,bits) (((val) & ((1 << (bits))-1)) << (shift)) 770 771 /* And this one does the opposite, i.e. extract a field's value from a bit pattern */ 772 #define FIELD_FROM_BITS(val,shift,bits) (((val) >> (shift)) & ((1 << (bits))-1)) 773 774 /* Mutex state: 775 * 776 * 0 for unlocked 777 * 1 for locked, no waiters 778 * 2 for locked, maybe waiters 779 */ 780 #define MUTEX_STATE_SHIFT 0 781 #define MUTEX_STATE_LEN 2 782 783 #define MUTEX_STATE_MASK FIELD_MASK(MUTEX_STATE_SHIFT, MUTEX_STATE_LEN) 784 #define MUTEX_STATE_FROM_BITS(v) FIELD_FROM_BITS(v, MUTEX_STATE_SHIFT, MUTEX_STATE_LEN) 785 #define MUTEX_STATE_TO_BITS(v) FIELD_TO_BITS(v, MUTEX_STATE_SHIFT, MUTEX_STATE_LEN) 786 787 #define MUTEX_STATE_UNLOCKED 0 /* must be 0 to match __PTHREAD_MUTEX_INIT_VALUE */ 788 #define MUTEX_STATE_LOCKED_UNCONTENDED 1 /* must be 1 due to atomic dec in unlock operation */ 789 #define MUTEX_STATE_LOCKED_CONTENDED 2 /* must be 1 + LOCKED_UNCONTENDED due to atomic dec */ 790 791 #define MUTEX_STATE_FROM_BITS(v) FIELD_FROM_BITS(v, MUTEX_STATE_SHIFT, MUTEX_STATE_LEN) 792 #define MUTEX_STATE_TO_BITS(v) FIELD_TO_BITS(v, MUTEX_STATE_SHIFT, MUTEX_STATE_LEN) 793 794 #define MUTEX_STATE_BITS_UNLOCKED MUTEX_STATE_TO_BITS(MUTEX_STATE_UNLOCKED) 795 #define MUTEX_STATE_BITS_LOCKED_UNCONTENDED MUTEX_STATE_TO_BITS(MUTEX_STATE_LOCKED_UNCONTENDED) 796 #define MUTEX_STATE_BITS_LOCKED_CONTENDED MUTEX_STATE_TO_BITS(MUTEX_STATE_LOCKED_CONTENDED) 797 798 /* return true iff the mutex if locked with no waiters */ 799 #define MUTEX_STATE_BITS_IS_LOCKED_UNCONTENDED(v) (((v) & MUTEX_STATE_MASK) == MUTEX_STATE_BITS_LOCKED_UNCONTENDED) 800 801 /* return true iff the mutex if locked with maybe waiters */ 802 #define MUTEX_STATE_BITS_IS_LOCKED_CONTENDED(v) (((v) & MUTEX_STATE_MASK) == MUTEX_STATE_BITS_LOCKED_CONTENDED) 803 804 /* used to flip from LOCKED_UNCONTENDED to LOCKED_CONTENDED */ 805 #define MUTEX_STATE_BITS_FLIP_CONTENTION(v) ((v) ^ (MUTEX_STATE_BITS_LOCKED_CONTENDED ^ MUTEX_STATE_BITS_LOCKED_UNCONTENDED)) 806 807 /* Mutex counter: 808 * 809 * We need to check for overflow before incrementing, and we also need to 810 * detect when the counter is 0 811 */ 812 #define MUTEX_COUNTER_SHIFT 2 813 #define MUTEX_COUNTER_LEN 11 814 #define MUTEX_COUNTER_MASK FIELD_MASK(MUTEX_COUNTER_SHIFT, MUTEX_COUNTER_LEN) 815 816 #define MUTEX_COUNTER_BITS_WILL_OVERFLOW(v) (((v) & MUTEX_COUNTER_MASK) == MUTEX_COUNTER_MASK) 817 #define MUTEX_COUNTER_BITS_IS_ZERO(v) (((v) & MUTEX_COUNTER_MASK) == 0) 818 819 /* Used to increment the counter directly after overflow has been checked */ 820 #define MUTEX_COUNTER_BITS_ONE FIELD_TO_BITS(1,MUTEX_COUNTER_SHIFT,MUTEX_COUNTER_LEN) 821 822 /* Returns true iff the counter is 0 */ 823 #define MUTEX_COUNTER_BITS_ARE_ZERO(v) (((v) & MUTEX_COUNTER_MASK) == 0) 824 825 /* Mutex shared bit flag 826 * 827 * This flag is set to indicate that the mutex is shared among processes. 828 * This changes the futex opcode we use for futex wait/wake operations 829 * (non-shared operations are much faster). 830 */ 831 #define MUTEX_SHARED_SHIFT 13 832 #define MUTEX_SHARED_MASK FIELD_MASK(MUTEX_SHARED_SHIFT,1) 833 834 /* Mutex type: 835 * 836 * We support normal, recursive and errorcheck mutexes. 837 * 838 * The constants defined here *cannot* be changed because they must match 839 * the C library ABI which defines the following initialization values in 840 * <pthread.h>: 841 * 842 * __PTHREAD_MUTEX_INIT_VALUE 843 * __PTHREAD_RECURSIVE_MUTEX_VALUE 844 * __PTHREAD_ERRORCHECK_MUTEX_INIT_VALUE 845 */ 846 #define MUTEX_TYPE_SHIFT 14 847 #define MUTEX_TYPE_LEN 2 848 #define MUTEX_TYPE_MASK FIELD_MASK(MUTEX_TYPE_SHIFT,MUTEX_TYPE_LEN) 849 850 #define MUTEX_TYPE_NORMAL 0 /* Must be 0 to match __PTHREAD_MUTEX_INIT_VALUE */ 851 #define MUTEX_TYPE_RECURSIVE 1 852 #define MUTEX_TYPE_ERRORCHECK 2 853 854 #define MUTEX_TYPE_TO_BITS(t) FIELD_TO_BITS(t, MUTEX_TYPE_SHIFT, MUTEX_TYPE_LEN) 855 856 #define MUTEX_TYPE_BITS_NORMAL MUTEX_TYPE_TO_BITS(MUTEX_TYPE_NORMAL) 857 #define MUTEX_TYPE_BITS_RECURSIVE MUTEX_TYPE_TO_BITS(MUTEX_TYPE_RECURSIVE) 858 #define MUTEX_TYPE_BITS_ERRORCHECK MUTEX_TYPE_TO_BITS(MUTEX_TYPE_ERRORCHECK) 859 860 /* Mutex owner field: 861 * 862 * This is only used for recursive and errorcheck mutexes. It holds the 863 * kernel TID of the owning thread. Note that this works because the Linux 864 * kernel _only_ uses 16-bit values for thread ids. 865 * 866 * More specifically, it will wrap to 10000 when it reaches over 32768 for 867 * application processes. You can check this by running the following inside 868 * an adb shell session: 869 * 870 OLDPID=$$; 871 while true; do 872 NEWPID=$(sh -c 'echo $$') 873 if [ "$NEWPID" -gt 32768 ]; then 874 echo "AARGH: new PID $NEWPID is too high!" 875 exit 1 876 fi 877 if [ "$NEWPID" -lt "$OLDPID" ]; then 878 echo "****** Wrapping from PID $OLDPID to $NEWPID. *******" 879 else 880 echo -n "$NEWPID!" 881 fi 882 OLDPID=$NEWPID 883 done 884 885 * Note that you can run the same example on a desktop Linux system, 886 * the wrapping will also happen at 32768, but will go back to 300 instead. 887 */ 888 #define MUTEX_OWNER_SHIFT 16 889 #define MUTEX_OWNER_LEN 16 890 891 #define MUTEX_OWNER_FROM_BITS(v) FIELD_FROM_BITS(v,MUTEX_OWNER_SHIFT,MUTEX_OWNER_LEN) 892 #define MUTEX_OWNER_TO_BITS(v) FIELD_TO_BITS(v,MUTEX_OWNER_SHIFT,MUTEX_OWNER_LEN) 893 894 /* Convenience macros. 895 * 896 * These are used to form or modify the bit pattern of a given mutex value 897 */ 898 899 900 901 /* a mutex attribute holds the following fields 902 * 903 * bits: name description 904 * 0-3 type type of mutex 905 * 4 shared process-shared flag 906 */ 907 #define MUTEXATTR_TYPE_MASK 0x000f 908 #define MUTEXATTR_SHARED_MASK 0x0010 909 910 911 int pthread_mutexattr_init(pthread_mutexattr_t *attr) 912 { 913 if (attr) { 914 *attr = PTHREAD_MUTEX_DEFAULT; 915 return 0; 916 } else { 917 return EINVAL; 918 } 919 } 920 921 int pthread_mutexattr_destroy(pthread_mutexattr_t *attr) 922 { 923 if (attr) { 924 *attr = -1; 925 return 0; 926 } else { 927 return EINVAL; 928 } 929 } 930 931 int pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *type) 932 { 933 if (attr) { 934 int atype = (*attr & MUTEXATTR_TYPE_MASK); 935 936 if (atype >= PTHREAD_MUTEX_NORMAL && 937 atype <= PTHREAD_MUTEX_ERRORCHECK) { 938 *type = atype; 939 return 0; 940 } 941 } 942 return EINVAL; 943 } 944 945 int pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type) 946 { 947 if (attr && type >= PTHREAD_MUTEX_NORMAL && 948 type <= PTHREAD_MUTEX_ERRORCHECK ) { 949 *attr = (*attr & ~MUTEXATTR_TYPE_MASK) | type; 950 return 0; 951 } 952 return EINVAL; 953 } 954 955 /* process-shared mutexes are not supported at the moment */ 956 957 int pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared) 958 { 959 if (!attr) 960 return EINVAL; 961 962 switch (pshared) { 963 case PTHREAD_PROCESS_PRIVATE: 964 *attr &= ~MUTEXATTR_SHARED_MASK; 965 return 0; 966 967 case PTHREAD_PROCESS_SHARED: 968 /* our current implementation of pthread actually supports shared 969 * mutexes but won't cleanup if a process dies with the mutex held. 970 * Nevertheless, it's better than nothing. Shared mutexes are used 971 * by surfaceflinger and audioflinger. 972 */ 973 *attr |= MUTEXATTR_SHARED_MASK; 974 return 0; 975 } 976 return EINVAL; 977 } 978 979 int pthread_mutexattr_getpshared(pthread_mutexattr_t *attr, int *pshared) 980 { 981 if (!attr || !pshared) 982 return EINVAL; 983 984 *pshared = (*attr & MUTEXATTR_SHARED_MASK) ? PTHREAD_PROCESS_SHARED 985 : PTHREAD_PROCESS_PRIVATE; 986 return 0; 987 } 988 989 int pthread_mutex_init(pthread_mutex_t *mutex, 990 const pthread_mutexattr_t *attr) 991 { 992 int value = 0; 993 994 if (mutex == NULL) 995 return EINVAL; 996 997 if (__likely(attr == NULL)) { 998 mutex->value = MUTEX_TYPE_BITS_NORMAL; 999 return 0; 1000 } 1001 1002 if ((*attr & MUTEXATTR_SHARED_MASK) != 0) 1003 value |= MUTEX_SHARED_MASK; 1004 1005 switch (*attr & MUTEXATTR_TYPE_MASK) { 1006 case PTHREAD_MUTEX_NORMAL: 1007 value |= MUTEX_TYPE_BITS_NORMAL; 1008 break; 1009 case PTHREAD_MUTEX_RECURSIVE: 1010 value |= MUTEX_TYPE_BITS_RECURSIVE; 1011 break; 1012 case PTHREAD_MUTEX_ERRORCHECK: 1013 value |= MUTEX_TYPE_BITS_ERRORCHECK; 1014 break; 1015 default: 1016 return EINVAL; 1017 } 1018 1019 mutex->value = value; 1020 return 0; 1021 } 1022 1023 1024 /* 1025 * Lock a non-recursive mutex. 1026 * 1027 * As noted above, there are three states: 1028 * 0 (unlocked, no contention) 1029 * 1 (locked, no contention) 1030 * 2 (locked, contention) 1031 * 1032 * Non-recursive mutexes don't use the thread-id or counter fields, and the 1033 * "type" value is zero, so the only bits that will be set are the ones in 1034 * the lock state field. 1035 */ 1036 static __inline__ void 1037 _normal_lock(pthread_mutex_t* mutex, int shared) 1038 { 1039 /* convenience shortcuts */ 1040 const int unlocked = shared | MUTEX_STATE_BITS_UNLOCKED; 1041 const int locked_uncontended = shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED; 1042 /* 1043 * The common case is an unlocked mutex, so we begin by trying to 1044 * change the lock's state from 0 (UNLOCKED) to 1 (LOCKED). 1045 * __bionic_cmpxchg() returns 0 if it made the swap successfully. 1046 * If the result is nonzero, this lock is already held by another thread. 1047 */ 1048 if (__bionic_cmpxchg(unlocked, locked_uncontended, &mutex->value) != 0) { 1049 const int locked_contended = shared | MUTEX_STATE_BITS_LOCKED_CONTENDED; 1050 /* 1051 * We want to go to sleep until the mutex is available, which 1052 * requires promoting it to state 2 (CONTENDED). We need to 1053 * swap in the new state value and then wait until somebody wakes us up. 1054 * 1055 * __bionic_swap() returns the previous value. We swap 2 in and 1056 * see if we got zero back; if so, we have acquired the lock. If 1057 * not, another thread still holds the lock and we wait again. 1058 * 1059 * The second argument to the __futex_wait() call is compared 1060 * against the current value. If it doesn't match, __futex_wait() 1061 * returns immediately (otherwise, it sleeps for a time specified 1062 * by the third argument; 0 means sleep forever). This ensures 1063 * that the mutex is in state 2 when we go to sleep on it, which 1064 * guarantees a wake-up call. 1065 */ 1066 while (__bionic_swap(locked_contended, &mutex->value) != unlocked) 1067 __futex_wait_ex(&mutex->value, shared, locked_contended, 0); 1068 } 1069 ANDROID_MEMBAR_FULL(); 1070 } 1071 1072 /* 1073 * Release a non-recursive mutex. The caller is responsible for determining 1074 * that we are in fact the owner of this lock. 1075 */ 1076 static __inline__ void 1077 _normal_unlock(pthread_mutex_t* mutex, int shared) 1078 { 1079 ANDROID_MEMBAR_FULL(); 1080 1081 /* 1082 * The mutex state will be 1 or (rarely) 2. We use an atomic decrement 1083 * to release the lock. __bionic_atomic_dec() returns the previous value; 1084 * if it wasn't 1 we have to do some additional work. 1085 */ 1086 if (__bionic_atomic_dec(&mutex->value) != (shared|MUTEX_STATE_BITS_LOCKED_UNCONTENDED)) { 1087 /* 1088 * Start by releasing the lock. The decrement changed it from 1089 * "contended lock" to "uncontended lock", which means we still 1090 * hold it, and anybody who tries to sneak in will push it back 1091 * to state 2. 1092 * 1093 * Once we set it to zero the lock is up for grabs. We follow 1094 * this with a __futex_wake() to ensure that one of the waiting 1095 * threads has a chance to grab it. 1096 * 1097 * This doesn't cause a race with the swap/wait pair in 1098 * _normal_lock(), because the __futex_wait() call there will 1099 * return immediately if the mutex value isn't 2. 1100 */ 1101 mutex->value = shared; 1102 1103 /* 1104 * Wake up one waiting thread. We don't know which thread will be 1105 * woken or when it'll start executing -- futexes make no guarantees 1106 * here. There may not even be a thread waiting. 1107 * 1108 * The newly-woken thread will replace the 0 we just set above 1109 * with 2, which means that when it eventually releases the mutex 1110 * it will also call FUTEX_WAKE. This results in one extra wake 1111 * call whenever a lock is contended, but lets us avoid forgetting 1112 * anyone without requiring us to track the number of sleepers. 1113 * 1114 * It's possible for another thread to sneak in and grab the lock 1115 * between the zero assignment above and the wake call below. If 1116 * the new thread is "slow" and holds the lock for a while, we'll 1117 * wake up a sleeper, which will swap in a 2 and then go back to 1118 * sleep since the lock is still held. If the new thread is "fast", 1119 * running to completion before we call wake, the thread we 1120 * eventually wake will find an unlocked mutex and will execute. 1121 * Either way we have correct behavior and nobody is orphaned on 1122 * the wait queue. 1123 */ 1124 __futex_wake_ex(&mutex->value, shared, 1); 1125 } 1126 } 1127 1128 /* This common inlined function is used to increment the counter of an 1129 * errorcheck or recursive mutex. 1130 * 1131 * For errorcheck mutexes, it will return EDEADLK 1132 * If the counter overflows, it will return EAGAIN 1133 * Otherwise, it atomically increments the counter and returns 0 1134 * after providing an acquire barrier. 1135 * 1136 * mtype is the current mutex type 1137 * mvalue is the current mutex value (already loaded) 1138 * mutex pointers to the mutex. 1139 */ 1140 static __inline__ __attribute__((always_inline)) int 1141 _recursive_increment(pthread_mutex_t* mutex, int mvalue, int mtype) 1142 { 1143 if (mtype == MUTEX_TYPE_BITS_ERRORCHECK) { 1144 /* trying to re-lock a mutex we already acquired */ 1145 return EDEADLK; 1146 } 1147 1148 /* Detect recursive lock overflow and return EAGAIN. 1149 * This is safe because only the owner thread can modify the 1150 * counter bits in the mutex value. 1151 */ 1152 if (MUTEX_COUNTER_BITS_WILL_OVERFLOW(mvalue)) { 1153 return EAGAIN; 1154 } 1155 1156 /* We own the mutex, but other threads are able to change 1157 * the lower bits (e.g. promoting it to "contended"), so we 1158 * need to use an atomic cmpxchg loop to update the counter. 1159 */ 1160 for (;;) { 1161 /* increment counter, overflow was already checked */ 1162 int newval = mvalue + MUTEX_COUNTER_BITS_ONE; 1163 if (__likely(__bionic_cmpxchg(mvalue, newval, &mutex->value) == 0)) { 1164 /* mutex is still locked, not need for a memory barrier */ 1165 return 0; 1166 } 1167 /* the value was changed, this happens when another thread changes 1168 * the lower state bits from 1 to 2 to indicate contention. This 1169 * cannot change the counter, so simply reload and try again. 1170 */ 1171 mvalue = mutex->value; 1172 } 1173 } 1174 1175 __LIBC_HIDDEN__ 1176 int pthread_mutex_lock_impl(pthread_mutex_t *mutex) 1177 { 1178 int mvalue, mtype, tid, new_lock_type, shared; 1179 1180 if (__unlikely(mutex == NULL)) 1181 return EINVAL; 1182 1183 mvalue = mutex->value; 1184 mtype = (mvalue & MUTEX_TYPE_MASK); 1185 shared = (mvalue & MUTEX_SHARED_MASK); 1186 1187 /* Handle normal case first */ 1188 if ( __likely(mtype == MUTEX_TYPE_BITS_NORMAL) ) { 1189 _normal_lock(mutex, shared); 1190 return 0; 1191 } 1192 1193 /* Do we already own this recursive or error-check mutex ? */ 1194 tid = __get_thread()->kernel_id; 1195 if ( tid == MUTEX_OWNER_FROM_BITS(mvalue) ) 1196 return _recursive_increment(mutex, mvalue, mtype); 1197 1198 /* Add in shared state to avoid extra 'or' operations below */ 1199 mtype |= shared; 1200 1201 /* First, if the mutex is unlocked, try to quickly acquire it. 1202 * In the optimistic case where this works, set the state to 1 to 1203 * indicate locked with no contention */ 1204 if (mvalue == mtype) { 1205 int newval = MUTEX_OWNER_TO_BITS(tid) | mtype | MUTEX_STATE_BITS_LOCKED_UNCONTENDED; 1206 if (__bionic_cmpxchg(mvalue, newval, &mutex->value) == 0) { 1207 ANDROID_MEMBAR_FULL(); 1208 return 0; 1209 } 1210 /* argh, the value changed, reload before entering the loop */ 1211 mvalue = mutex->value; 1212 } 1213 1214 for (;;) { 1215 int newval; 1216 1217 /* if the mutex is unlocked, its value should be 'mtype' and 1218 * we try to acquire it by setting its owner and state atomically. 1219 * NOTE: We put the state to 2 since we _know_ there is contention 1220 * when we are in this loop. This ensures all waiters will be 1221 * unlocked. 1222 */ 1223 if (mvalue == mtype) { 1224 newval = MUTEX_OWNER_TO_BITS(tid) | mtype | MUTEX_STATE_BITS_LOCKED_CONTENDED; 1225 /* TODO: Change this to __bionic_cmpxchg_acquire when we 1226 * implement it to get rid of the explicit memory 1227 * barrier below. 1228 */ 1229 if (__unlikely(__bionic_cmpxchg(mvalue, newval, &mutex->value) != 0)) { 1230 mvalue = mutex->value; 1231 continue; 1232 } 1233 ANDROID_MEMBAR_FULL(); 1234 return 0; 1235 } 1236 1237 /* the mutex is already locked by another thread, if its state is 1 1238 * we will change it to 2 to indicate contention. */ 1239 if (MUTEX_STATE_BITS_IS_LOCKED_UNCONTENDED(mvalue)) { 1240 newval = MUTEX_STATE_BITS_FLIP_CONTENTION(mvalue); /* locked state 1 => state 2 */ 1241 if (__unlikely(__bionic_cmpxchg(mvalue, newval, &mutex->value) != 0)) { 1242 mvalue = mutex->value; 1243 continue; 1244 } 1245 mvalue = newval; 1246 } 1247 1248 /* wait until the mutex is unlocked */ 1249 __futex_wait_ex(&mutex->value, shared, mvalue, NULL); 1250 1251 mvalue = mutex->value; 1252 } 1253 /* NOTREACHED */ 1254 } 1255 1256 int pthread_mutex_lock(pthread_mutex_t *mutex) 1257 { 1258 int err = pthread_mutex_lock_impl(mutex); 1259 #ifdef PTHREAD_DEBUG 1260 if (PTHREAD_DEBUG_ENABLED) { 1261 if (!err) { 1262 pthread_debug_mutex_lock_check(mutex); 1263 } 1264 } 1265 #endif 1266 return err; 1267 } 1268 1269 __LIBC_HIDDEN__ 1270 int pthread_mutex_unlock_impl(pthread_mutex_t *mutex) 1271 { 1272 int mvalue, mtype, tid, oldv, shared; 1273 1274 if (__unlikely(mutex == NULL)) 1275 return EINVAL; 1276 1277 mvalue = mutex->value; 1278 mtype = (mvalue & MUTEX_TYPE_MASK); 1279 shared = (mvalue & MUTEX_SHARED_MASK); 1280 1281 /* Handle common case first */ 1282 if (__likely(mtype == MUTEX_TYPE_BITS_NORMAL)) { 1283 _normal_unlock(mutex, shared); 1284 return 0; 1285 } 1286 1287 /* Do we already own this recursive or error-check mutex ? */ 1288 tid = __get_thread()->kernel_id; 1289 if ( tid != MUTEX_OWNER_FROM_BITS(mvalue) ) 1290 return EPERM; 1291 1292 /* If the counter is > 0, we can simply decrement it atomically. 1293 * Since other threads can mutate the lower state bits (and only the 1294 * lower state bits), use a cmpxchg to do it. 1295 */ 1296 if (!MUTEX_COUNTER_BITS_IS_ZERO(mvalue)) { 1297 for (;;) { 1298 int newval = mvalue - MUTEX_COUNTER_BITS_ONE; 1299 if (__likely(__bionic_cmpxchg(mvalue, newval, &mutex->value) == 0)) { 1300 /* success: we still own the mutex, so no memory barrier */ 1301 return 0; 1302 } 1303 /* the value changed, so reload and loop */ 1304 mvalue = mutex->value; 1305 } 1306 } 1307 1308 /* the counter is 0, so we're going to unlock the mutex by resetting 1309 * its value to 'unlocked'. We need to perform a swap in order 1310 * to read the current state, which will be 2 if there are waiters 1311 * to awake. 1312 * 1313 * TODO: Change this to __bionic_swap_release when we implement it 1314 * to get rid of the explicit memory barrier below. 1315 */ 1316 ANDROID_MEMBAR_FULL(); /* RELEASE BARRIER */ 1317 mvalue = __bionic_swap(mtype | shared | MUTEX_STATE_BITS_UNLOCKED, &mutex->value); 1318 1319 /* Wake one waiting thread, if any */ 1320 if (MUTEX_STATE_BITS_IS_LOCKED_CONTENDED(mvalue)) { 1321 __futex_wake_ex(&mutex->value, shared, 1); 1322 } 1323 return 0; 1324 } 1325 1326 int pthread_mutex_unlock(pthread_mutex_t *mutex) 1327 { 1328 #ifdef PTHREAD_DEBUG 1329 if (PTHREAD_DEBUG_ENABLED) { 1330 pthread_debug_mutex_unlock_check(mutex); 1331 } 1332 #endif 1333 return pthread_mutex_unlock_impl(mutex); 1334 } 1335 1336 __LIBC_HIDDEN__ 1337 int pthread_mutex_trylock_impl(pthread_mutex_t *mutex) 1338 { 1339 int mvalue, mtype, tid, oldv, shared; 1340 1341 if (__unlikely(mutex == NULL)) 1342 return EINVAL; 1343 1344 mvalue = mutex->value; 1345 mtype = (mvalue & MUTEX_TYPE_MASK); 1346 shared = (mvalue & MUTEX_SHARED_MASK); 1347 1348 /* Handle common case first */ 1349 if ( __likely(mtype == MUTEX_TYPE_BITS_NORMAL) ) 1350 { 1351 if (__bionic_cmpxchg(shared|MUTEX_STATE_BITS_UNLOCKED, 1352 shared|MUTEX_STATE_BITS_LOCKED_UNCONTENDED, 1353 &mutex->value) == 0) { 1354 ANDROID_MEMBAR_FULL(); 1355 return 0; 1356 } 1357 1358 return EBUSY; 1359 } 1360 1361 /* Do we already own this recursive or error-check mutex ? */ 1362 tid = __get_thread()->kernel_id; 1363 if ( tid == MUTEX_OWNER_FROM_BITS(mvalue) ) 1364 return _recursive_increment(mutex, mvalue, mtype); 1365 1366 /* Same as pthread_mutex_lock, except that we don't want to wait, and 1367 * the only operation that can succeed is a single cmpxchg to acquire the 1368 * lock if it is released / not owned by anyone. No need for a complex loop. 1369 */ 1370 mtype |= shared | MUTEX_STATE_BITS_UNLOCKED; 1371 mvalue = MUTEX_OWNER_TO_BITS(tid) | mtype | MUTEX_STATE_BITS_LOCKED_UNCONTENDED; 1372 1373 if (__likely(__bionic_cmpxchg(mtype, mvalue, &mutex->value) == 0)) { 1374 ANDROID_MEMBAR_FULL(); 1375 return 0; 1376 } 1377 1378 return EBUSY; 1379 } 1380 1381 int pthread_mutex_trylock(pthread_mutex_t *mutex) 1382 { 1383 int err = pthread_mutex_trylock_impl(mutex); 1384 #ifdef PTHREAD_DEBUG 1385 if (PTHREAD_DEBUG_ENABLED) { 1386 if (!err) { 1387 pthread_debug_mutex_lock_check(mutex); 1388 } 1389 } 1390 #endif 1391 return err; 1392 } 1393 1394 /* initialize 'ts' with the difference between 'abstime' and the current time 1395 * according to 'clock'. Returns -1 if abstime already expired, or 0 otherwise. 1396 */ 1397 static int 1398 __timespec_to_absolute(struct timespec* ts, const struct timespec* abstime, clockid_t clock) 1399 { 1400 clock_gettime(clock, ts); 1401 ts->tv_sec = abstime->tv_sec - ts->tv_sec; 1402 ts->tv_nsec = abstime->tv_nsec - ts->tv_nsec; 1403 if (ts->tv_nsec < 0) { 1404 ts->tv_sec--; 1405 ts->tv_nsec += 1000000000; 1406 } 1407 if ((ts->tv_nsec < 0) || (ts->tv_sec < 0)) 1408 return -1; 1409 1410 return 0; 1411 } 1412 1413 /* initialize 'abstime' to the current time according to 'clock' plus 'msecs' 1414 * milliseconds. 1415 */ 1416 static void 1417 __timespec_to_relative_msec(struct timespec* abstime, unsigned msecs, clockid_t clock) 1418 { 1419 clock_gettime(clock, abstime); 1420 abstime->tv_sec += msecs/1000; 1421 abstime->tv_nsec += (msecs%1000)*1000000; 1422 if (abstime->tv_nsec >= 1000000000) { 1423 abstime->tv_sec++; 1424 abstime->tv_nsec -= 1000000000; 1425 } 1426 } 1427 1428 __LIBC_HIDDEN__ 1429 int pthread_mutex_lock_timeout_np_impl(pthread_mutex_t *mutex, unsigned msecs) 1430 { 1431 clockid_t clock = CLOCK_MONOTONIC; 1432 struct timespec abstime; 1433 struct timespec ts; 1434 int mvalue, mtype, tid, oldv, new_lock_type, shared; 1435 1436 /* compute absolute expiration time */ 1437 __timespec_to_relative_msec(&abstime, msecs, clock); 1438 1439 if (__unlikely(mutex == NULL)) 1440 return EINVAL; 1441 1442 mvalue = mutex->value; 1443 mtype = (mvalue & MUTEX_TYPE_MASK); 1444 shared = (mvalue & MUTEX_SHARED_MASK); 1445 1446 /* Handle common case first */ 1447 if ( __likely(mtype == MUTEX_TYPE_BITS_NORMAL) ) 1448 { 1449 const int unlocked = shared | MUTEX_STATE_BITS_UNLOCKED; 1450 const int locked_uncontended = shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED; 1451 const int locked_contended = shared | MUTEX_STATE_BITS_LOCKED_CONTENDED; 1452 1453 /* fast path for uncontended lock. Note: MUTEX_TYPE_BITS_NORMAL is 0 */ 1454 if (__bionic_cmpxchg(unlocked, locked_uncontended, &mutex->value) == 0) { 1455 ANDROID_MEMBAR_FULL(); 1456 return 0; 1457 } 1458 1459 /* loop while needed */ 1460 while (__bionic_swap(locked_contended, &mutex->value) != unlocked) { 1461 if (__timespec_to_absolute(&ts, &abstime, clock) < 0) 1462 return EBUSY; 1463 1464 __futex_wait_ex(&mutex->value, shared, locked_contended, &ts); 1465 } 1466 ANDROID_MEMBAR_FULL(); 1467 return 0; 1468 } 1469 1470 /* Do we already own this recursive or error-check mutex ? */ 1471 tid = __get_thread()->kernel_id; 1472 if ( tid == MUTEX_OWNER_FROM_BITS(mvalue) ) 1473 return _recursive_increment(mutex, mvalue, mtype); 1474 1475 /* the following implements the same loop than pthread_mutex_lock_impl 1476 * but adds checks to ensure that the operation never exceeds the 1477 * absolute expiration time. 1478 */ 1479 mtype |= shared; 1480 1481 /* first try a quick lock */ 1482 if (mvalue == mtype) { 1483 mvalue = MUTEX_OWNER_TO_BITS(tid) | mtype | MUTEX_STATE_BITS_LOCKED_UNCONTENDED; 1484 if (__likely(__bionic_cmpxchg(mtype, mvalue, &mutex->value) == 0)) { 1485 ANDROID_MEMBAR_FULL(); 1486 return 0; 1487 } 1488 mvalue = mutex->value; 1489 } 1490 1491 for (;;) { 1492 struct timespec ts; 1493 1494 /* if the value is 'unlocked', try to acquire it directly */ 1495 /* NOTE: put state to 2 since we know there is contention */ 1496 if (mvalue == mtype) /* unlocked */ { 1497 mvalue = MUTEX_OWNER_TO_BITS(tid) | mtype | MUTEX_STATE_BITS_LOCKED_CONTENDED; 1498 if (__bionic_cmpxchg(mtype, mvalue, &mutex->value) == 0) { 1499 ANDROID_MEMBAR_FULL(); 1500 return 0; 1501 } 1502 /* the value changed before we could lock it. We need to check 1503 * the time to avoid livelocks, reload the value, then loop again. */ 1504 if (__timespec_to_absolute(&ts, &abstime, clock) < 0) 1505 return EBUSY; 1506 1507 mvalue = mutex->value; 1508 continue; 1509 } 1510 1511 /* The value is locked. If 'uncontended', try to switch its state 1512 * to 'contented' to ensure we get woken up later. */ 1513 if (MUTEX_STATE_BITS_IS_LOCKED_UNCONTENDED(mvalue)) { 1514 int newval = MUTEX_STATE_BITS_FLIP_CONTENTION(mvalue); 1515 if (__bionic_cmpxchg(mvalue, newval, &mutex->value) != 0) { 1516 /* this failed because the value changed, reload it */ 1517 mvalue = mutex->value; 1518 } else { 1519 /* this succeeded, update mvalue */ 1520 mvalue = newval; 1521 } 1522 } 1523 1524 /* check time and update 'ts' */ 1525 if (__timespec_to_absolute(&ts, &abstime, clock) < 0) 1526 return EBUSY; 1527 1528 /* Only wait to be woken up if the state is '2', otherwise we'll 1529 * simply loop right now. This can happen when the second cmpxchg 1530 * in our loop failed because the mutex was unlocked by another 1531 * thread. 1532 */ 1533 if (MUTEX_STATE_BITS_IS_LOCKED_CONTENDED(mvalue)) { 1534 if (__futex_wait_ex(&mutex->value, shared, mvalue, &ts) == ETIMEDOUT) { 1535 return EBUSY; 1536 } 1537 mvalue = mutex->value; 1538 } 1539 } 1540 /* NOTREACHED */ 1541 } 1542 1543 int pthread_mutex_lock_timeout_np(pthread_mutex_t *mutex, unsigned msecs) 1544 { 1545 int err = pthread_mutex_lock_timeout_np_impl(mutex, msecs); 1546 #ifdef PTHREAD_DEBUG 1547 if (PTHREAD_DEBUG_ENABLED) { 1548 if (!err) { 1549 pthread_debug_mutex_lock_check(mutex); 1550 } 1551 } 1552 #endif 1553 return err; 1554 } 1555 1556 int pthread_mutex_destroy(pthread_mutex_t *mutex) 1557 { 1558 int ret; 1559 1560 /* use trylock to ensure that the mutex value is 1561 * valid and is not already locked. */ 1562 ret = pthread_mutex_trylock_impl(mutex); 1563 if (ret != 0) 1564 return ret; 1565 1566 mutex->value = 0xdead10cc; 1567 return 0; 1568 } 1569 1570 1571 1572 int pthread_condattr_init(pthread_condattr_t *attr) 1573 { 1574 if (attr == NULL) 1575 return EINVAL; 1576 1577 *attr = PTHREAD_PROCESS_PRIVATE; 1578 return 0; 1579 } 1580 1581 int pthread_condattr_getpshared(pthread_condattr_t *attr, int *pshared) 1582 { 1583 if (attr == NULL || pshared == NULL) 1584 return EINVAL; 1585 1586 *pshared = *attr; 1587 return 0; 1588 } 1589 1590 int pthread_condattr_setpshared(pthread_condattr_t *attr, int pshared) 1591 { 1592 if (attr == NULL) 1593 return EINVAL; 1594 1595 if (pshared != PTHREAD_PROCESS_SHARED && 1596 pshared != PTHREAD_PROCESS_PRIVATE) 1597 return EINVAL; 1598 1599 *attr = pshared; 1600 return 0; 1601 } 1602 1603 int pthread_condattr_destroy(pthread_condattr_t *attr) 1604 { 1605 if (attr == NULL) 1606 return EINVAL; 1607 1608 *attr = 0xdeada11d; 1609 return 0; 1610 } 1611 1612 /* We use one bit in condition variable values as the 'shared' flag 1613 * The rest is a counter. 1614 */ 1615 #define COND_SHARED_MASK 0x0001 1616 #define COND_COUNTER_INCREMENT 0x0002 1617 #define COND_COUNTER_MASK (~COND_SHARED_MASK) 1618 1619 #define COND_IS_SHARED(c) (((c)->value & COND_SHARED_MASK) != 0) 1620 1621 /* XXX *technically* there is a race condition that could allow 1622 * XXX a signal to be missed. If thread A is preempted in _wait() 1623 * XXX after unlocking the mutex and before waiting, and if other 1624 * XXX threads call signal or broadcast UINT_MAX/2 times (exactly), 1625 * XXX before thread A is scheduled again and calls futex_wait(), 1626 * XXX then the signal will be lost. 1627 */ 1628 1629 int pthread_cond_init(pthread_cond_t *cond, 1630 const pthread_condattr_t *attr) 1631 { 1632 if (cond == NULL) 1633 return EINVAL; 1634 1635 cond->value = 0; 1636 1637 if (attr != NULL && *attr == PTHREAD_PROCESS_SHARED) 1638 cond->value |= COND_SHARED_MASK; 1639 1640 return 0; 1641 } 1642 1643 int pthread_cond_destroy(pthread_cond_t *cond) 1644 { 1645 if (cond == NULL) 1646 return EINVAL; 1647 1648 cond->value = 0xdeadc04d; 1649 return 0; 1650 } 1651 1652 /* This function is used by pthread_cond_broadcast and 1653 * pthread_cond_signal to atomically decrement the counter 1654 * then wake-up 'counter' threads. 1655 */ 1656 static int 1657 __pthread_cond_pulse(pthread_cond_t *cond, int counter) 1658 { 1659 long flags; 1660 1661 if (__unlikely(cond == NULL)) 1662 return EINVAL; 1663 1664 flags = (cond->value & ~COND_COUNTER_MASK); 1665 for (;;) { 1666 long oldval = cond->value; 1667 long newval = ((oldval - COND_COUNTER_INCREMENT) & COND_COUNTER_MASK) 1668 | flags; 1669 if (__bionic_cmpxchg(oldval, newval, &cond->value) == 0) 1670 break; 1671 } 1672 1673 /* 1674 * Ensure that all memory accesses previously made by this thread are 1675 * visible to the woken thread(s). On the other side, the "wait" 1676 * code will issue any necessary barriers when locking the mutex. 1677 * 1678 * This may not strictly be necessary -- if the caller follows 1679 * recommended practice and holds the mutex before signaling the cond 1680 * var, the mutex ops will provide correct semantics. If they don't 1681 * hold the mutex, they're subject to race conditions anyway. 1682 */ 1683 ANDROID_MEMBAR_FULL(); 1684 1685 __futex_wake_ex(&cond->value, COND_IS_SHARED(cond), counter); 1686 return 0; 1687 } 1688 1689 int pthread_cond_broadcast(pthread_cond_t *cond) 1690 { 1691 return __pthread_cond_pulse(cond, INT_MAX); 1692 } 1693 1694 int pthread_cond_signal(pthread_cond_t *cond) 1695 { 1696 return __pthread_cond_pulse(cond, 1); 1697 } 1698 1699 int pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex) 1700 { 1701 return pthread_cond_timedwait(cond, mutex, NULL); 1702 } 1703 1704 int __pthread_cond_timedwait_relative(pthread_cond_t *cond, 1705 pthread_mutex_t * mutex, 1706 const struct timespec *reltime) 1707 { 1708 int status; 1709 int oldvalue = cond->value; 1710 1711 pthread_mutex_unlock(mutex); 1712 status = __futex_wait_ex(&cond->value, COND_IS_SHARED(cond), oldvalue, reltime); 1713 pthread_mutex_lock(mutex); 1714 1715 if (status == (-ETIMEDOUT)) return ETIMEDOUT; 1716 return 0; 1717 } 1718 1719 int __pthread_cond_timedwait(pthread_cond_t *cond, 1720 pthread_mutex_t * mutex, 1721 const struct timespec *abstime, 1722 clockid_t clock) 1723 { 1724 struct timespec ts; 1725 struct timespec * tsp; 1726 1727 if (abstime != NULL) { 1728 if (__timespec_to_absolute(&ts, abstime, clock) < 0) 1729 return ETIMEDOUT; 1730 tsp = &ts; 1731 } else { 1732 tsp = NULL; 1733 } 1734 1735 return __pthread_cond_timedwait_relative(cond, mutex, tsp); 1736 } 1737 1738 int pthread_cond_timedwait(pthread_cond_t *cond, 1739 pthread_mutex_t * mutex, 1740 const struct timespec *abstime) 1741 { 1742 return __pthread_cond_timedwait(cond, mutex, abstime, CLOCK_REALTIME); 1743 } 1744 1745 1746 /* this one exists only for backward binary compatibility */ 1747 int pthread_cond_timedwait_monotonic(pthread_cond_t *cond, 1748 pthread_mutex_t * mutex, 1749 const struct timespec *abstime) 1750 { 1751 return __pthread_cond_timedwait(cond, mutex, abstime, CLOCK_MONOTONIC); 1752 } 1753 1754 int pthread_cond_timedwait_monotonic_np(pthread_cond_t *cond, 1755 pthread_mutex_t * mutex, 1756 const struct timespec *abstime) 1757 { 1758 return __pthread_cond_timedwait(cond, mutex, abstime, CLOCK_MONOTONIC); 1759 } 1760 1761 int pthread_cond_timedwait_relative_np(pthread_cond_t *cond, 1762 pthread_mutex_t * mutex, 1763 const struct timespec *reltime) 1764 { 1765 return __pthread_cond_timedwait_relative(cond, mutex, reltime); 1766 } 1767 1768 int pthread_cond_timeout_np(pthread_cond_t *cond, 1769 pthread_mutex_t * mutex, 1770 unsigned msecs) 1771 { 1772 struct timespec ts; 1773 1774 ts.tv_sec = msecs / 1000; 1775 ts.tv_nsec = (msecs % 1000) * 1000000; 1776 1777 return __pthread_cond_timedwait_relative(cond, mutex, &ts); 1778 } 1779 1780 1781 1782 /* A technical note regarding our thread-local-storage (TLS) implementation: 1783 * 1784 * There can be up to TLSMAP_SIZE independent TLS keys in a given process, 1785 * though the first TLSMAP_START keys are reserved for Bionic to hold 1786 * special thread-specific variables like errno or a pointer to 1787 * the current thread's descriptor. 1788 * 1789 * while stored in the TLS area, these entries cannot be accessed through 1790 * pthread_getspecific() / pthread_setspecific() and pthread_key_delete() 1791 * 1792 * also, some entries in the key table are pre-allocated (see tlsmap_lock) 1793 * to greatly simplify and speedup some OpenGL-related operations. though the 1794 * initialy value will be NULL on all threads. 1795 * 1796 * you can use pthread_getspecific()/setspecific() on these, and in theory 1797 * you could also call pthread_key_delete() as well, though this would 1798 * probably break some apps. 1799 * 1800 * The 'tlsmap_t' type defined below implements a shared global map of 1801 * currently created/allocated TLS keys and the destructors associated 1802 * with them. You should use tlsmap_lock/unlock to access it to avoid 1803 * any race condition. 1804 * 1805 * the global TLS map simply contains a bitmap of allocated keys, and 1806 * an array of destructors. 1807 * 1808 * each thread has a TLS area that is a simple array of TLSMAP_SIZE void* 1809 * pointers. the TLS area of the main thread is stack-allocated in 1810 * __libc_init_common, while the TLS area of other threads is placed at 1811 * the top of their stack in pthread_create. 1812 * 1813 * when pthread_key_create() is called, it finds the first free key in the 1814 * bitmap, then set it to 1, saving the destructor altogether 1815 * 1816 * when pthread_key_delete() is called. it will erase the key's bitmap bit 1817 * and its destructor, and will also clear the key data in the TLS area of 1818 * all created threads. As mandated by Posix, it is the responsability of 1819 * the caller of pthread_key_delete() to properly reclaim the objects that 1820 * were pointed to by these data fields (either before or after the call). 1821 * 1822 */ 1823 1824 /* TLS Map implementation 1825 */ 1826 1827 #define TLSMAP_START (TLS_SLOT_MAX_WELL_KNOWN+1) 1828 #define TLSMAP_SIZE BIONIC_TLS_SLOTS 1829 #define TLSMAP_BITS 32 1830 #define TLSMAP_WORDS ((TLSMAP_SIZE+TLSMAP_BITS-1)/TLSMAP_BITS) 1831 #define TLSMAP_WORD(m,k) (m)->map[(k)/TLSMAP_BITS] 1832 #define TLSMAP_MASK(k) (1U << ((k)&(TLSMAP_BITS-1))) 1833 1834 /* this macro is used to quickly check that a key belongs to a reasonable range */ 1835 #define TLSMAP_VALIDATE_KEY(key) \ 1836 ((key) >= TLSMAP_START && (key) < TLSMAP_SIZE) 1837 1838 /* the type of tls key destructor functions */ 1839 typedef void (*tls_dtor_t)(void*); 1840 1841 typedef struct { 1842 int init; /* see comment in tlsmap_lock() */ 1843 uint32_t map[TLSMAP_WORDS]; /* bitmap of allocated keys */ 1844 tls_dtor_t dtors[TLSMAP_SIZE]; /* key destructors */ 1845 } tlsmap_t; 1846 1847 static pthread_mutex_t _tlsmap_lock = PTHREAD_MUTEX_INITIALIZER; 1848 static tlsmap_t _tlsmap; 1849 1850 /* lock the global TLS map lock and return a handle to it */ 1851 static __inline__ tlsmap_t* tlsmap_lock(void) 1852 { 1853 tlsmap_t* m = &_tlsmap; 1854 1855 pthread_mutex_lock(&_tlsmap_lock); 1856 /* we need to initialize the first entry of the 'map' array 1857 * with the value TLS_DEFAULT_ALLOC_MAP. doing it statically 1858 * when declaring _tlsmap is a bit awkward and is going to 1859 * produce warnings, so do it the first time we use the map 1860 * instead 1861 */ 1862 if (__unlikely(!m->init)) { 1863 TLSMAP_WORD(m,0) = TLS_DEFAULT_ALLOC_MAP; 1864 m->init = 1; 1865 } 1866 return m; 1867 } 1868 1869 /* unlock the global TLS map */ 1870 static __inline__ void tlsmap_unlock(tlsmap_t* m) 1871 { 1872 pthread_mutex_unlock(&_tlsmap_lock); 1873 (void)m; /* a good compiler is a happy compiler */ 1874 } 1875 1876 /* test to see wether a key is allocated */ 1877 static __inline__ int tlsmap_test(tlsmap_t* m, int key) 1878 { 1879 return (TLSMAP_WORD(m,key) & TLSMAP_MASK(key)) != 0; 1880 } 1881 1882 /* set the destructor and bit flag on a newly allocated key */ 1883 static __inline__ void tlsmap_set(tlsmap_t* m, int key, tls_dtor_t dtor) 1884 { 1885 TLSMAP_WORD(m,key) |= TLSMAP_MASK(key); 1886 m->dtors[key] = dtor; 1887 } 1888 1889 /* clear the destructor and bit flag on an existing key */ 1890 static __inline__ void tlsmap_clear(tlsmap_t* m, int key) 1891 { 1892 TLSMAP_WORD(m,key) &= ~TLSMAP_MASK(key); 1893 m->dtors[key] = NULL; 1894 } 1895 1896 /* allocate a new TLS key, return -1 if no room left */ 1897 static int tlsmap_alloc(tlsmap_t* m, tls_dtor_t dtor) 1898 { 1899 int key; 1900 1901 for ( key = TLSMAP_START; key < TLSMAP_SIZE; key++ ) { 1902 if ( !tlsmap_test(m, key) ) { 1903 tlsmap_set(m, key, dtor); 1904 return key; 1905 } 1906 } 1907 return -1; 1908 } 1909 1910 1911 int pthread_key_create(pthread_key_t *key, void (*destructor_function)(void *)) 1912 { 1913 uint32_t err = ENOMEM; 1914 tlsmap_t* map = tlsmap_lock(); 1915 int k = tlsmap_alloc(map, destructor_function); 1916 1917 if (k >= 0) { 1918 *key = k; 1919 err = 0; 1920 } 1921 tlsmap_unlock(map); 1922 return err; 1923 } 1924 1925 1926 /* This deletes a pthread_key_t. note that the standard mandates that this does 1927 * not call the destructor of non-NULL key values. Instead, it is the 1928 * responsability of the caller to properly dispose of the corresponding data 1929 * and resources, using any mean it finds suitable. 1930 * 1931 * On the other hand, this function will clear the corresponding key data 1932 * values in all known threads. this prevents later (invalid) calls to 1933 * pthread_getspecific() to receive invalid/stale values. 1934 */ 1935 int pthread_key_delete(pthread_key_t key) 1936 { 1937 uint32_t err; 1938 pthread_internal_t* thr; 1939 tlsmap_t* map; 1940 1941 if (!TLSMAP_VALIDATE_KEY(key)) { 1942 return EINVAL; 1943 } 1944 1945 map = tlsmap_lock(); 1946 1947 if (!tlsmap_test(map, key)) { 1948 err = EINVAL; 1949 goto err1; 1950 } 1951 1952 /* clear value in all threads */ 1953 pthread_mutex_lock(&gThreadListLock); 1954 for ( thr = gThreadList; thr != NULL; thr = thr->next ) { 1955 /* avoid zombie threads with a negative 'join_count'. these are really 1956 * already dead and don't have a TLS area anymore. 1957 * 1958 * similarly, it is possible to have thr->tls == NULL for threads that 1959 * were just recently created through pthread_create() but whose 1960 * startup trampoline (__thread_entry) hasn't been run yet by the 1961 * scheduler. so check for this too. 1962 */ 1963 if (thr->join_count < 0 || !thr->tls) 1964 continue; 1965 1966 thr->tls[key] = NULL; 1967 } 1968 tlsmap_clear(map, key); 1969 1970 pthread_mutex_unlock(&gThreadListLock); 1971 err = 0; 1972 1973 err1: 1974 tlsmap_unlock(map); 1975 return err; 1976 } 1977 1978 1979 int pthread_setspecific(pthread_key_t key, const void *ptr) 1980 { 1981 int err = EINVAL; 1982 tlsmap_t* map; 1983 1984 if (TLSMAP_VALIDATE_KEY(key)) { 1985 /* check that we're trying to set data for an allocated key */ 1986 map = tlsmap_lock(); 1987 if (tlsmap_test(map, key)) { 1988 ((uint32_t *)__get_tls())[key] = (uint32_t)ptr; 1989 err = 0; 1990 } 1991 tlsmap_unlock(map); 1992 } 1993 return err; 1994 } 1995 1996 void * pthread_getspecific(pthread_key_t key) 1997 { 1998 if (!TLSMAP_VALIDATE_KEY(key)) { 1999 return NULL; 2000 } 2001 2002 /* for performance reason, we do not lock/unlock the global TLS map 2003 * to check that the key is properly allocated. if the key was not 2004 * allocated, the value read from the TLS should always be NULL 2005 * due to pthread_key_delete() clearing the values for all threads. 2006 */ 2007 return (void *)(((unsigned *)__get_tls())[key]); 2008 } 2009 2010 /* Posix mandates that this be defined in <limits.h> but we don't have 2011 * it just yet. 2012 */ 2013 #ifndef PTHREAD_DESTRUCTOR_ITERATIONS 2014 # define PTHREAD_DESTRUCTOR_ITERATIONS 4 2015 #endif 2016 2017 /* this function is called from pthread_exit() to remove all TLS key data 2018 * from this thread's TLS area. this must call the destructor of all keys 2019 * that have a non-NULL data value (and a non-NULL destructor). 2020 * 2021 * because destructors can do funky things like deleting/creating other 2022 * keys, we need to implement this in a loop 2023 */ 2024 static void pthread_key_clean_all(void) 2025 { 2026 tlsmap_t* map; 2027 void** tls = (void**)__get_tls(); 2028 int rounds = PTHREAD_DESTRUCTOR_ITERATIONS; 2029 2030 map = tlsmap_lock(); 2031 2032 for (rounds = PTHREAD_DESTRUCTOR_ITERATIONS; rounds > 0; rounds--) 2033 { 2034 int kk, count = 0; 2035 2036 for (kk = TLSMAP_START; kk < TLSMAP_SIZE; kk++) { 2037 if ( tlsmap_test(map, kk) ) 2038 { 2039 void* data = tls[kk]; 2040 tls_dtor_t dtor = map->dtors[kk]; 2041 2042 if (data != NULL && dtor != NULL) 2043 { 2044 /* we need to clear the key data now, this will prevent the 2045 * destructor (or a later one) from seeing the old value if 2046 * it calls pthread_getspecific() for some odd reason 2047 * 2048 * we do not do this if 'dtor == NULL' just in case another 2049 * destructor function might be responsible for manually 2050 * releasing the corresponding data. 2051 */ 2052 tls[kk] = NULL; 2053 2054 /* because the destructor is free to call pthread_key_create 2055 * and/or pthread_key_delete, we need to temporarily unlock 2056 * the TLS map 2057 */ 2058 tlsmap_unlock(map); 2059 (*dtor)(data); 2060 map = tlsmap_lock(); 2061 2062 count += 1; 2063 } 2064 } 2065 } 2066 2067 /* if we didn't call any destructor, there is no need to check the 2068 * TLS data again 2069 */ 2070 if (count == 0) 2071 break; 2072 } 2073 tlsmap_unlock(map); 2074 } 2075 2076 // man says this should be in <linux/unistd.h>, but it isn't 2077 extern int tgkill(int tgid, int tid, int sig); 2078 2079 int pthread_kill(pthread_t tid, int sig) 2080 { 2081 int ret; 2082 int old_errno = errno; 2083 pthread_internal_t * thread = (pthread_internal_t *)tid; 2084 2085 ret = tgkill(getpid(), thread->kernel_id, sig); 2086 if (ret < 0) { 2087 ret = errno; 2088 errno = old_errno; 2089 } 2090 2091 return ret; 2092 } 2093 2094 /* Despite the fact that our kernel headers define sigset_t explicitly 2095 * as a 32-bit integer, the kernel system call really expects a 64-bit 2096 * bitmap for the signal set, or more exactly an array of two-32-bit 2097 * values (see $KERNEL/arch/$ARCH/include/asm/signal.h for details). 2098 * 2099 * Unfortunately, we cannot fix the sigset_t definition without breaking 2100 * the C library ABI, so perform a little runtime translation here. 2101 */ 2102 typedef union { 2103 sigset_t bionic; 2104 uint32_t kernel[2]; 2105 } kernel_sigset_t; 2106 2107 /* this is a private syscall stub */ 2108 extern int __rt_sigprocmask(int, const kernel_sigset_t *, kernel_sigset_t *, size_t); 2109 2110 int pthread_sigmask(int how, const sigset_t *set, sigset_t *oset) 2111 { 2112 /* pthread_sigmask must return the error code, but the syscall 2113 * will set errno instead and return 0/-1 2114 */ 2115 int ret, old_errno = errno; 2116 2117 /* We must convert *set into a kernel_sigset_t */ 2118 kernel_sigset_t in_set, *in_set_ptr; 2119 kernel_sigset_t out_set; 2120 2121 in_set.kernel[0] = in_set.kernel[1] = 0; 2122 out_set.kernel[0] = out_set.kernel[1] = 0; 2123 2124 /* 'in_set_ptr' is the second parameter to __rt_sigprocmask. It must be NULL 2125 * if 'set' is NULL to ensure correct semantics (which in this case would 2126 * be to ignore 'how' and return the current signal set into 'oset'. 2127 */ 2128 if (set == NULL) { 2129 in_set_ptr = NULL; 2130 } else { 2131 in_set.bionic = *set; 2132 in_set_ptr = &in_set; 2133 } 2134 2135 ret = __rt_sigprocmask(how, in_set_ptr, &out_set, sizeof(kernel_sigset_t)); 2136 if (ret < 0) 2137 ret = errno; 2138 2139 if (oset) 2140 *oset = out_set.bionic; 2141 2142 errno = old_errno; 2143 return ret; 2144 } 2145 2146 2147 int pthread_getcpuclockid(pthread_t tid, clockid_t *clockid) 2148 { 2149 const int CLOCK_IDTYPE_BITS = 3; 2150 pthread_internal_t* thread = (pthread_internal_t*)tid; 2151 2152 if (!thread) 2153 return ESRCH; 2154 2155 *clockid = CLOCK_THREAD_CPUTIME_ID | (thread->kernel_id << CLOCK_IDTYPE_BITS); 2156 return 0; 2157 } 2158 2159 2160 /* NOTE: this implementation doesn't support a init function that throws a C++ exception 2161 * or calls fork() 2162 */ 2163 int pthread_once( pthread_once_t* once_control, void (*init_routine)(void) ) 2164 { 2165 static pthread_mutex_t once_lock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER; 2166 volatile pthread_once_t* ocptr = once_control; 2167 pthread_once_t value; 2168 2169 /* PTHREAD_ONCE_INIT is 0, we use the following bit flags 2170 * 2171 * bit 0 set -> initialization is under way 2172 * bit 1 set -> initialization is complete 2173 */ 2174 #define ONCE_INITIALIZING (1 << 0) 2175 #define ONCE_COMPLETED (1 << 1) 2176 2177 /* First check if the once is already initialized. This will be the common 2178 * case and we want to make this as fast as possible. Note that this still 2179 * requires a load_acquire operation here to ensure that all the 2180 * stores performed by the initialization function are observable on 2181 * this CPU after we exit. 2182 */ 2183 if (__likely((*ocptr & ONCE_COMPLETED) != 0)) { 2184 ANDROID_MEMBAR_FULL(); 2185 return 0; 2186 } 2187 2188 for (;;) { 2189 /* Try to atomically set the INITIALIZING flag. 2190 * This requires a cmpxchg loop, and we may need 2191 * to exit prematurely if we detect that 2192 * COMPLETED is now set. 2193 */ 2194 int32_t oldval, newval; 2195 2196 do { 2197 oldval = *ocptr; 2198 if ((oldval & ONCE_COMPLETED) != 0) 2199 break; 2200 2201 newval = oldval | ONCE_INITIALIZING; 2202 } while (__bionic_cmpxchg(oldval, newval, ocptr) != 0); 2203 2204 if ((oldval & ONCE_COMPLETED) != 0) { 2205 /* We detected that COMPLETED was set while in our loop */ 2206 ANDROID_MEMBAR_FULL(); 2207 return 0; 2208 } 2209 2210 if ((oldval & ONCE_INITIALIZING) == 0) { 2211 /* We got there first, we can jump out of the loop to 2212 * handle the initialization */ 2213 break; 2214 } 2215 2216 /* Another thread is running the initialization and hasn't completed 2217 * yet, so wait for it, then try again. */ 2218 __futex_wait_ex(ocptr, 0, oldval, NULL); 2219 } 2220 2221 /* call the initialization function. */ 2222 (*init_routine)(); 2223 2224 /* Do a store_release indicating that initialization is complete */ 2225 ANDROID_MEMBAR_FULL(); 2226 *ocptr = ONCE_COMPLETED; 2227 2228 /* Wake up any waiters, if any */ 2229 __futex_wake_ex(ocptr, 0, INT_MAX); 2230 2231 return 0; 2232 } 2233 2234 /* This value is not exported by kernel headers, so hardcode it here */ 2235 #define MAX_TASK_COMM_LEN 16 2236 #define TASK_COMM_FMT "/proc/self/task/%u/comm" 2237 2238 int pthread_setname_np(pthread_t thid, const char *thname) 2239 { 2240 size_t thname_len; 2241 int saved_errno, ret; 2242 2243 if (thid == 0 || thname == NULL) 2244 return EINVAL; 2245 2246 thname_len = strlen(thname); 2247 if (thname_len >= MAX_TASK_COMM_LEN) 2248 return ERANGE; 2249 2250 saved_errno = errno; 2251 if (thid == pthread_self()) 2252 { 2253 ret = prctl(PR_SET_NAME, (unsigned long)thname, 0, 0, 0) ? errno : 0; 2254 } 2255 else 2256 { 2257 /* Have to change another thread's name */ 2258 pthread_internal_t *thread = (pthread_internal_t *)thid; 2259 char comm_name[sizeof(TASK_COMM_FMT) + 8]; 2260 ssize_t n; 2261 int fd; 2262 2263 snprintf(comm_name, sizeof(comm_name), TASK_COMM_FMT, (unsigned int)thread->kernel_id); 2264 fd = open(comm_name, O_RDWR); 2265 if (fd == -1) 2266 { 2267 ret = errno; 2268 goto exit; 2269 } 2270 n = TEMP_FAILURE_RETRY(write(fd, thname, thname_len)); 2271 close(fd); 2272 2273 if (n < 0) 2274 ret = errno; 2275 else if ((size_t)n != thname_len) 2276 ret = EIO; 2277 else 2278 ret = 0; 2279 } 2280 exit: 2281 errno = saved_errno; 2282 return ret; 2283 } 2284 2285 /* Return the kernel thread ID for a pthread. 2286 * This is only defined for implementations where pthread <-> kernel is 1:1, which this is. 2287 * Not the same as pthread_getthreadid_np, which is commonly defined to be opaque. 2288 * Internal, not an NDK API. 2289 */ 2290 2291 pid_t __pthread_gettid(pthread_t thid) 2292 { 2293 pthread_internal_t* thread = (pthread_internal_t*)thid; 2294 return thread->kernel_id; 2295 } 2296 2297 int __pthread_settid(pthread_t thid, pid_t tid) 2298 { 2299 if (thid == 0) 2300 return EINVAL; 2301 2302 pthread_internal_t* thread = (pthread_internal_t*)thid; 2303 thread->kernel_id = tid; 2304 2305 return 0; 2306 } 2307
