1 /* 2 * Copyright (C) 2008 The Android Open Source Project 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * * Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * * Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in 12 * the documentation and/or other materials provided with the 13 * distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 18 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 19 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 22 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 25 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <pthread.h> 30 31 #include <errno.h> 32 #include <limits.h> 33 #include <stdatomic.h> 34 #include <string.h> 35 #include <sys/cdefs.h> 36 #include <sys/mman.h> 37 #include <unistd.h> 38 39 #include "pthread_internal.h" 40 41 #include "private/bionic_constants.h" 42 #include "private/bionic_futex.h" 43 #include "private/bionic_systrace.h" 44 #include "private/bionic_time_conversions.h" 45 #include "private/bionic_tls.h" 46 47 /* a mutex attribute holds the following fields 48 * 49 * bits: name description 50 * 0-3 type type of mutex 51 * 4 shared process-shared flag 52 */ 53 #define MUTEXATTR_TYPE_MASK 0x000f 54 #define MUTEXATTR_SHARED_MASK 0x0010 55 56 int pthread_mutexattr_init(pthread_mutexattr_t *attr) 57 { 58 *attr = PTHREAD_MUTEX_DEFAULT; 59 return 0; 60 } 61 62 int pthread_mutexattr_destroy(pthread_mutexattr_t *attr) 63 { 64 *attr = -1; 65 return 0; 66 } 67 68 int pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *type_p) 69 { 70 int type = (*attr & MUTEXATTR_TYPE_MASK); 71 72 if (type < PTHREAD_MUTEX_NORMAL || type > PTHREAD_MUTEX_ERRORCHECK) { 73 return EINVAL; 74 } 75 76 *type_p = type; 77 return 0; 78 } 79 80 int pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type) 81 { 82 if (type < PTHREAD_MUTEX_NORMAL || type > PTHREAD_MUTEX_ERRORCHECK ) { 83 return EINVAL; 84 } 85 86 *attr = (*attr & ~MUTEXATTR_TYPE_MASK) | type; 87 return 0; 88 } 89 90 /* process-shared mutexes are not supported at the moment */ 91 92 int pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared) 93 { 94 switch (pshared) { 95 case PTHREAD_PROCESS_PRIVATE: 96 *attr &= ~MUTEXATTR_SHARED_MASK; 97 return 0; 98 99 case PTHREAD_PROCESS_SHARED: 100 /* our current implementation of pthread actually supports shared 101 * mutexes but won't cleanup if a process dies with the mutex held. 102 * Nevertheless, it's better than nothing. Shared mutexes are used 103 * by surfaceflinger and audioflinger. 104 */ 105 *attr |= MUTEXATTR_SHARED_MASK; 106 return 0; 107 } 108 return EINVAL; 109 } 110 111 int pthread_mutexattr_getpshared(const pthread_mutexattr_t* attr, int* pshared) { 112 *pshared = (*attr & MUTEXATTR_SHARED_MASK) ? PTHREAD_PROCESS_SHARED : PTHREAD_PROCESS_PRIVATE; 113 return 0; 114 } 115 116 /* a mutex contains a state value and a owner_tid. 117 * The value is implemented as a 16-bit integer holding the following fields: 118 * 119 * bits: name description 120 * 15-14 type mutex type 121 * 13 shared process-shared flag 122 * 12-2 counter counter of recursive mutexes 123 * 1-0 state lock state (0, 1 or 2) 124 * 125 * The owner_tid is used only in recursive and errorcheck mutex to hold the mutex owner thread tid. 126 */ 127 128 /* Convenience macro, creates a mask of 'bits' bits that starts from 129 * the 'shift'-th least significant bit in a 32-bit word. 130 * 131 * Examples: FIELD_MASK(0,4) -> 0xf 132 * FIELD_MASK(16,9) -> 0x1ff0000 133 */ 134 #define FIELD_MASK(shift,bits) (((1 << (bits))-1) << (shift)) 135 136 /* This one is used to create a bit pattern from a given field value */ 137 #define FIELD_TO_BITS(val,shift,bits) (((val) & ((1 << (bits))-1)) << (shift)) 138 139 /* And this one does the opposite, i.e. extract a field's value from a bit pattern */ 140 #define FIELD_FROM_BITS(val,shift,bits) (((val) >> (shift)) & ((1 << (bits))-1)) 141 142 143 /* Convenience macros. 144 * 145 * These are used to form or modify the bit pattern of a given mutex value 146 */ 147 148 /* Mutex state: 149 * 150 * 0 for unlocked 151 * 1 for locked, no waiters 152 * 2 for locked, maybe waiters 153 */ 154 #define MUTEX_STATE_SHIFT 0 155 #define MUTEX_STATE_LEN 2 156 157 #define MUTEX_STATE_MASK FIELD_MASK(MUTEX_STATE_SHIFT, MUTEX_STATE_LEN) 158 #define MUTEX_STATE_FROM_BITS(v) FIELD_FROM_BITS(v, MUTEX_STATE_SHIFT, MUTEX_STATE_LEN) 159 #define MUTEX_STATE_TO_BITS(v) FIELD_TO_BITS(v, MUTEX_STATE_SHIFT, MUTEX_STATE_LEN) 160 161 #define MUTEX_STATE_UNLOCKED 0 /* must be 0 to match PTHREAD_MUTEX_INITIALIZER */ 162 #define MUTEX_STATE_LOCKED_UNCONTENDED 1 /* must be 1 due to atomic dec in unlock operation */ 163 #define MUTEX_STATE_LOCKED_CONTENDED 2 /* must be 1 + LOCKED_UNCONTENDED due to atomic dec */ 164 165 #define MUTEX_STATE_BITS_UNLOCKED MUTEX_STATE_TO_BITS(MUTEX_STATE_UNLOCKED) 166 #define MUTEX_STATE_BITS_LOCKED_UNCONTENDED MUTEX_STATE_TO_BITS(MUTEX_STATE_LOCKED_UNCONTENDED) 167 #define MUTEX_STATE_BITS_LOCKED_CONTENDED MUTEX_STATE_TO_BITS(MUTEX_STATE_LOCKED_CONTENDED) 168 169 // Return true iff the mutex is unlocked. 170 #define MUTEX_STATE_BITS_IS_UNLOCKED(v) (((v) & MUTEX_STATE_MASK) == MUTEX_STATE_BITS_UNLOCKED) 171 172 // Return true iff the mutex is locked with no waiters. 173 #define MUTEX_STATE_BITS_IS_LOCKED_UNCONTENDED(v) (((v) & MUTEX_STATE_MASK) == MUTEX_STATE_BITS_LOCKED_UNCONTENDED) 174 175 // return true iff the mutex is locked with maybe waiters. 176 #define MUTEX_STATE_BITS_IS_LOCKED_CONTENDED(v) (((v) & MUTEX_STATE_MASK) == MUTEX_STATE_BITS_LOCKED_CONTENDED) 177 178 /* used to flip from LOCKED_UNCONTENDED to LOCKED_CONTENDED */ 179 #define MUTEX_STATE_BITS_FLIP_CONTENTION(v) ((v) ^ (MUTEX_STATE_BITS_LOCKED_CONTENDED ^ MUTEX_STATE_BITS_LOCKED_UNCONTENDED)) 180 181 /* Mutex counter: 182 * 183 * We need to check for overflow before incrementing, and we also need to 184 * detect when the counter is 0 185 */ 186 #define MUTEX_COUNTER_SHIFT 2 187 #define MUTEX_COUNTER_LEN 11 188 #define MUTEX_COUNTER_MASK FIELD_MASK(MUTEX_COUNTER_SHIFT, MUTEX_COUNTER_LEN) 189 190 #define MUTEX_COUNTER_BITS_WILL_OVERFLOW(v) (((v) & MUTEX_COUNTER_MASK) == MUTEX_COUNTER_MASK) 191 #define MUTEX_COUNTER_BITS_IS_ZERO(v) (((v) & MUTEX_COUNTER_MASK) == 0) 192 193 /* Used to increment the counter directly after overflow has been checked */ 194 #define MUTEX_COUNTER_BITS_ONE FIELD_TO_BITS(1, MUTEX_COUNTER_SHIFT,MUTEX_COUNTER_LEN) 195 196 /* Mutex shared bit flag 197 * 198 * This flag is set to indicate that the mutex is shared among processes. 199 * This changes the futex opcode we use for futex wait/wake operations 200 * (non-shared operations are much faster). 201 */ 202 #define MUTEX_SHARED_SHIFT 13 203 #define MUTEX_SHARED_MASK FIELD_MASK(MUTEX_SHARED_SHIFT,1) 204 205 /* Mutex type: 206 * We support normal, recursive and errorcheck mutexes. 207 */ 208 #define MUTEX_TYPE_SHIFT 14 209 #define MUTEX_TYPE_LEN 2 210 #define MUTEX_TYPE_MASK FIELD_MASK(MUTEX_TYPE_SHIFT,MUTEX_TYPE_LEN) 211 212 #define MUTEX_TYPE_TO_BITS(t) FIELD_TO_BITS(t, MUTEX_TYPE_SHIFT, MUTEX_TYPE_LEN) 213 214 #define MUTEX_TYPE_BITS_NORMAL MUTEX_TYPE_TO_BITS(PTHREAD_MUTEX_NORMAL) 215 #define MUTEX_TYPE_BITS_RECURSIVE MUTEX_TYPE_TO_BITS(PTHREAD_MUTEX_RECURSIVE) 216 #define MUTEX_TYPE_BITS_ERRORCHECK MUTEX_TYPE_TO_BITS(PTHREAD_MUTEX_ERRORCHECK) 217 218 struct pthread_mutex_internal_t { 219 _Atomic(uint16_t) state; 220 #if defined(__LP64__) 221 uint16_t __pad; 222 atomic_int owner_tid; 223 char __reserved[32]; 224 #else 225 _Atomic(uint16_t) owner_tid; 226 #endif 227 } __attribute__((aligned(4))); 228 229 static_assert(sizeof(pthread_mutex_t) == sizeof(pthread_mutex_internal_t), 230 "pthread_mutex_t should actually be pthread_mutex_internal_t in implementation."); 231 232 // For binary compatibility with old version of pthread_mutex_t, we can't use more strict alignment 233 // than 4-byte alignment. 234 static_assert(alignof(pthread_mutex_t) == 4, 235 "pthread_mutex_t should fulfill the alignment of pthread_mutex_internal_t."); 236 237 static inline pthread_mutex_internal_t* __get_internal_mutex(pthread_mutex_t* mutex_interface) { 238 return reinterpret_cast<pthread_mutex_internal_t*>(mutex_interface); 239 } 240 241 int pthread_mutex_init(pthread_mutex_t* mutex_interface, const pthread_mutexattr_t* attr) { 242 pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); 243 244 memset(mutex, 0, sizeof(pthread_mutex_internal_t)); 245 246 if (__predict_true(attr == NULL)) { 247 atomic_init(&mutex->state, MUTEX_TYPE_BITS_NORMAL); 248 return 0; 249 } 250 251 uint16_t state = 0; 252 if ((*attr & MUTEXATTR_SHARED_MASK) != 0) { 253 state |= MUTEX_SHARED_MASK; 254 } 255 256 switch (*attr & MUTEXATTR_TYPE_MASK) { 257 case PTHREAD_MUTEX_NORMAL: 258 state |= MUTEX_TYPE_BITS_NORMAL; 259 break; 260 case PTHREAD_MUTEX_RECURSIVE: 261 state |= MUTEX_TYPE_BITS_RECURSIVE; 262 break; 263 case PTHREAD_MUTEX_ERRORCHECK: 264 state |= MUTEX_TYPE_BITS_ERRORCHECK; 265 break; 266 default: 267 return EINVAL; 268 } 269 270 atomic_init(&mutex->state, state); 271 atomic_init(&mutex->owner_tid, 0); 272 return 0; 273 } 274 275 static inline __always_inline int __pthread_normal_mutex_trylock(pthread_mutex_internal_t* mutex, 276 uint16_t shared) { 277 const uint16_t unlocked = shared | MUTEX_STATE_BITS_UNLOCKED; 278 const uint16_t locked_uncontended = shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED; 279 280 uint16_t old_state = unlocked; 281 if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex->state, &old_state, 282 locked_uncontended, memory_order_acquire, memory_order_relaxed))) { 283 return 0; 284 } 285 return EBUSY; 286 } 287 288 /* 289 * Lock a mutex of type NORMAL. 290 * 291 * As noted above, there are three states: 292 * 0 (unlocked, no contention) 293 * 1 (locked, no contention) 294 * 2 (locked, contention) 295 * 296 * Non-recursive mutexes don't use the thread-id or counter fields, and the 297 * "type" value is zero, so the only bits that will be set are the ones in 298 * the lock state field. 299 */ 300 static inline __always_inline int __pthread_normal_mutex_lock(pthread_mutex_internal_t* mutex, 301 uint16_t shared, 302 bool use_realtime_clock, 303 const timespec* abs_timeout_or_null) { 304 if (__predict_true(__pthread_normal_mutex_trylock(mutex, shared) == 0)) { 305 return 0; 306 } 307 int result = check_timespec(abs_timeout_or_null, true); 308 if (result != 0) { 309 return result; 310 } 311 312 ScopedTrace trace("Contending for pthread mutex"); 313 314 const uint16_t unlocked = shared | MUTEX_STATE_BITS_UNLOCKED; 315 const uint16_t locked_contended = shared | MUTEX_STATE_BITS_LOCKED_CONTENDED; 316 317 // We want to go to sleep until the mutex is available, which requires 318 // promoting it to locked_contended. We need to swap in the new state 319 // and then wait until somebody wakes us up. 320 // An atomic_exchange is used to compete with other threads for the lock. 321 // If it returns unlocked, we have acquired the lock, otherwise another 322 // thread still holds the lock and we should wait again. 323 // If lock is acquired, an acquire fence is needed to make all memory accesses 324 // made by other threads visible to the current CPU. 325 while (atomic_exchange_explicit(&mutex->state, locked_contended, 326 memory_order_acquire) != unlocked) { 327 if (__futex_wait_ex(&mutex->state, shared, locked_contended, use_realtime_clock, 328 abs_timeout_or_null) == -ETIMEDOUT) { 329 return ETIMEDOUT; 330 } 331 } 332 return 0; 333 } 334 335 /* 336 * Release a normal mutex. The caller is responsible for determining 337 * that we are in fact the owner of this lock. 338 */ 339 static inline __always_inline void __pthread_normal_mutex_unlock(pthread_mutex_internal_t* mutex, 340 uint16_t shared) { 341 const uint16_t unlocked = shared | MUTEX_STATE_BITS_UNLOCKED; 342 const uint16_t locked_contended = shared | MUTEX_STATE_BITS_LOCKED_CONTENDED; 343 344 // We use an atomic_exchange to release the lock. If locked_contended state 345 // is returned, some threads is waiting for the lock and we need to wake up 346 // one of them. 347 // A release fence is required to make previous stores visible to next 348 // lock owner threads. 349 if (atomic_exchange_explicit(&mutex->state, unlocked, 350 memory_order_release) == locked_contended) { 351 // Wake up one waiting thread. We don't know which thread will be 352 // woken or when it'll start executing -- futexes make no guarantees 353 // here. There may not even be a thread waiting. 354 // 355 // The newly-woken thread will replace the unlocked state we just set above 356 // with locked_contended state, which means that when it eventually releases 357 // the mutex it will also call FUTEX_WAKE. This results in one extra wake 358 // call whenever a lock is contended, but let us avoid forgetting anyone 359 // without requiring us to track the number of sleepers. 360 // 361 // It's possible for another thread to sneak in and grab the lock between 362 // the exchange above and the wake call below. If the new thread is "slow" 363 // and holds the lock for a while, we'll wake up a sleeper, which will swap 364 // in locked_uncontended state and then go back to sleep since the lock is 365 // still held. If the new thread is "fast", running to completion before 366 // we call wake, the thread we eventually wake will find an unlocked mutex 367 // and will execute. Either way we have correct behavior and nobody is 368 // orphaned on the wait queue. 369 __futex_wake_ex(&mutex->state, shared, 1); 370 } 371 } 372 373 /* This common inlined function is used to increment the counter of a recursive mutex. 374 * 375 * If the counter overflows, it will return EAGAIN. 376 * Otherwise, it atomically increments the counter and returns 0. 377 * 378 */ 379 static inline __always_inline int __recursive_increment(pthread_mutex_internal_t* mutex, 380 uint16_t old_state) { 381 // Detect recursive lock overflow and return EAGAIN. 382 // This is safe because only the owner thread can modify the 383 // counter bits in the mutex value. 384 if (MUTEX_COUNTER_BITS_WILL_OVERFLOW(old_state)) { 385 return EAGAIN; 386 } 387 388 // Other threads are able to change the lower bits (e.g. promoting it to "contended"), 389 // but the mutex counter will not overflow. So we use atomic_fetch_add operation here. 390 // The mutex is still locked by current thread, so we don't need a release fence. 391 atomic_fetch_add_explicit(&mutex->state, MUTEX_COUNTER_BITS_ONE, memory_order_relaxed); 392 return 0; 393 } 394 395 static inline __always_inline int __recursive_or_errorcheck_mutex_wait( 396 pthread_mutex_internal_t* mutex, 397 uint16_t shared, 398 uint16_t old_state, 399 bool use_realtime_clock, 400 const timespec* abs_timeout) { 401 // __futex_wait always waits on a 32-bit value. But state is 16-bit. For a normal mutex, the owner_tid 402 // field in mutex is not used. On 64-bit devices, the __pad field in mutex is not used. 403 // But when a recursive or errorcheck mutex is used on 32-bit devices, we need to add the 404 // owner_tid value in the value argument for __futex_wait, otherwise we may always get EAGAIN error. 405 406 #if defined(__LP64__) 407 return __futex_wait_ex(&mutex->state, shared, old_state, use_realtime_clock, abs_timeout); 408 409 #else 410 // This implementation works only when the layout of pthread_mutex_internal_t matches below expectation. 411 // And it is based on the assumption that Android is always in little-endian devices. 412 static_assert(offsetof(pthread_mutex_internal_t, state) == 0, ""); 413 static_assert(offsetof(pthread_mutex_internal_t, owner_tid) == 2, ""); 414 415 uint32_t owner_tid = atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed); 416 return __futex_wait_ex(&mutex->state, shared, (owner_tid << 16) | old_state, 417 use_realtime_clock, abs_timeout); 418 #endif 419 } 420 421 static int __pthread_mutex_lock_with_timeout(pthread_mutex_internal_t* mutex, 422 bool use_realtime_clock, 423 const timespec* abs_timeout_or_null) { 424 uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); 425 uint16_t mtype = (old_state & MUTEX_TYPE_MASK); 426 uint16_t shared = (old_state & MUTEX_SHARED_MASK); 427 428 // Handle common case first. 429 if ( __predict_true(mtype == MUTEX_TYPE_BITS_NORMAL) ) { 430 return __pthread_normal_mutex_lock(mutex, shared, use_realtime_clock, abs_timeout_or_null); 431 } 432 433 // Do we already own this recursive or error-check mutex? 434 pid_t tid = __get_thread()->tid; 435 if (tid == atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed)) { 436 if (mtype == MUTEX_TYPE_BITS_ERRORCHECK) { 437 return EDEADLK; 438 } 439 return __recursive_increment(mutex, old_state); 440 } 441 442 const uint16_t unlocked = mtype | shared | MUTEX_STATE_BITS_UNLOCKED; 443 const uint16_t locked_uncontended = mtype | shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED; 444 const uint16_t locked_contended = mtype | shared | MUTEX_STATE_BITS_LOCKED_CONTENDED; 445 446 // First, if the mutex is unlocked, try to quickly acquire it. 447 // In the optimistic case where this works, set the state to locked_uncontended. 448 if (old_state == unlocked) { 449 // If exchanged successfully, an acquire fence is required to make 450 // all memory accesses made by other threads visible to the current CPU. 451 if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex->state, &old_state, 452 locked_uncontended, memory_order_acquire, memory_order_relaxed))) { 453 atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed); 454 return 0; 455 } 456 } 457 458 ScopedTrace trace("Contending for pthread mutex"); 459 460 while (true) { 461 if (old_state == unlocked) { 462 // NOTE: We put the state to locked_contended since we _know_ there 463 // is contention when we are in this loop. This ensures all waiters 464 // will be unlocked. 465 466 // If exchanged successfully, an acquire fence is required to make 467 // all memory accesses made by other threads visible to the current CPU. 468 if (__predict_true(atomic_compare_exchange_weak_explicit(&mutex->state, 469 &old_state, locked_contended, 470 memory_order_acquire, 471 memory_order_relaxed))) { 472 atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed); 473 return 0; 474 } 475 continue; 476 } else if (MUTEX_STATE_BITS_IS_LOCKED_UNCONTENDED(old_state)) { 477 // We should set it to locked_contended beforing going to sleep. This can make 478 // sure waiters will be woken up eventually. 479 480 int new_state = MUTEX_STATE_BITS_FLIP_CONTENTION(old_state); 481 if (__predict_false(!atomic_compare_exchange_weak_explicit(&mutex->state, 482 &old_state, new_state, 483 memory_order_relaxed, 484 memory_order_relaxed))) { 485 continue; 486 } 487 old_state = new_state; 488 } 489 490 int result = check_timespec(abs_timeout_or_null, true); 491 if (result != 0) { 492 return result; 493 } 494 // We are in locked_contended state, sleep until someone wakes us up. 495 if (__recursive_or_errorcheck_mutex_wait(mutex, shared, old_state, use_realtime_clock, 496 abs_timeout_or_null) == -ETIMEDOUT) { 497 return ETIMEDOUT; 498 } 499 old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); 500 } 501 } 502 503 int pthread_mutex_lock(pthread_mutex_t* mutex_interface) { 504 #if !defined(__LP64__) 505 // Some apps depend on being able to pass NULL as a mutex and get EINVAL 506 // back. Don't need to worry about it for LP64 since the ABI is brand new, 507 // but keep compatibility for LP32. http://b/19995172. 508 if (mutex_interface == NULL) { 509 return EINVAL; 510 } 511 #endif 512 513 pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); 514 515 uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); 516 uint16_t mtype = (old_state & MUTEX_TYPE_MASK); 517 uint16_t shared = (old_state & MUTEX_SHARED_MASK); 518 // Avoid slowing down fast path of normal mutex lock operation. 519 if (__predict_true(mtype == MUTEX_TYPE_BITS_NORMAL)) { 520 if (__predict_true(__pthread_normal_mutex_trylock(mutex, shared) == 0)) { 521 return 0; 522 } 523 } 524 return __pthread_mutex_lock_with_timeout(mutex, false, nullptr); 525 } 526 527 int pthread_mutex_unlock(pthread_mutex_t* mutex_interface) { 528 #if !defined(__LP64__) 529 // Some apps depend on being able to pass NULL as a mutex and get EINVAL 530 // back. Don't need to worry about it for LP64 since the ABI is brand new, 531 // but keep compatibility for LP32. http://b/19995172. 532 if (mutex_interface == NULL) { 533 return EINVAL; 534 } 535 #endif 536 537 pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); 538 539 uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); 540 uint16_t mtype = (old_state & MUTEX_TYPE_MASK); 541 uint16_t shared = (old_state & MUTEX_SHARED_MASK); 542 543 // Handle common case first. 544 if (__predict_true(mtype == MUTEX_TYPE_BITS_NORMAL)) { 545 __pthread_normal_mutex_unlock(mutex, shared); 546 return 0; 547 } 548 549 // Do we already own this recursive or error-check mutex? 550 pid_t tid = __get_thread()->tid; 551 if ( tid != atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed) ) { 552 return EPERM; 553 } 554 555 // If the counter is > 0, we can simply decrement it atomically. 556 // Since other threads can mutate the lower state bits (and only the 557 // lower state bits), use a compare_exchange loop to do it. 558 if (!MUTEX_COUNTER_BITS_IS_ZERO(old_state)) { 559 // We still own the mutex, so a release fence is not needed. 560 atomic_fetch_sub_explicit(&mutex->state, MUTEX_COUNTER_BITS_ONE, memory_order_relaxed); 561 return 0; 562 } 563 564 // The counter is 0, so we'are going to unlock the mutex by resetting its 565 // state to unlocked, we need to perform a atomic_exchange inorder to read 566 // the current state, which will be locked_contended if there may have waiters 567 // to awake. 568 // A release fence is required to make previous stores visible to next 569 // lock owner threads. 570 atomic_store_explicit(&mutex->owner_tid, 0, memory_order_relaxed); 571 const uint16_t unlocked = mtype | shared | MUTEX_STATE_BITS_UNLOCKED; 572 old_state = atomic_exchange_explicit(&mutex->state, unlocked, memory_order_release); 573 if (MUTEX_STATE_BITS_IS_LOCKED_CONTENDED(old_state)) { 574 __futex_wake_ex(&mutex->state, shared, 1); 575 } 576 577 return 0; 578 } 579 580 int pthread_mutex_trylock(pthread_mutex_t* mutex_interface) { 581 pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); 582 583 uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); 584 uint16_t mtype = (old_state & MUTEX_TYPE_MASK); 585 uint16_t shared = (old_state & MUTEX_SHARED_MASK); 586 587 const uint16_t unlocked = mtype | shared | MUTEX_STATE_BITS_UNLOCKED; 588 const uint16_t locked_uncontended = mtype | shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED; 589 590 // Handle common case first. 591 if (__predict_true(mtype == MUTEX_TYPE_BITS_NORMAL)) { 592 return __pthread_normal_mutex_trylock(mutex, shared); 593 } 594 595 // Do we already own this recursive or error-check mutex? 596 pid_t tid = __get_thread()->tid; 597 if (tid == atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed)) { 598 if (mtype == MUTEX_TYPE_BITS_ERRORCHECK) { 599 return EBUSY; 600 } 601 return __recursive_increment(mutex, old_state); 602 } 603 604 // Same as pthread_mutex_lock, except that we don't want to wait, and 605 // the only operation that can succeed is a single compare_exchange to acquire the 606 // lock if it is released / not owned by anyone. No need for a complex loop. 607 // If exchanged successfully, an acquire fence is required to make 608 // all memory accesses made by other threads visible to the current CPU. 609 old_state = unlocked; 610 if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex->state, &old_state, 611 locked_uncontended, 612 memory_order_acquire, 613 memory_order_relaxed))) { 614 atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed); 615 return 0; 616 } 617 return EBUSY; 618 } 619 620 #if !defined(__LP64__) 621 extern "C" int pthread_mutex_lock_timeout_np(pthread_mutex_t* mutex_interface, unsigned ms) { 622 timespec ts; 623 timespec_from_ms(ts, ms); 624 timespec abs_timeout; 625 absolute_timespec_from_timespec(abs_timeout, ts, CLOCK_MONOTONIC); 626 int error = __pthread_mutex_lock_with_timeout(__get_internal_mutex(mutex_interface), 627 false, &abs_timeout); 628 if (error == ETIMEDOUT) { 629 error = EBUSY; 630 } 631 return error; 632 } 633 #endif 634 635 int pthread_mutex_timedlock(pthread_mutex_t* mutex_interface, const timespec* abs_timeout) { 636 return __pthread_mutex_lock_with_timeout(__get_internal_mutex(mutex_interface), 637 true, abs_timeout); 638 } 639 640 int pthread_mutex_destroy(pthread_mutex_t* mutex_interface) { 641 pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); 642 uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); 643 // Store 0xffff to make the mutex unusable. Although POSIX standard says it is undefined 644 // behavior to destroy a locked mutex, we prefer not to change mutex->state in that situation. 645 if (MUTEX_STATE_BITS_IS_UNLOCKED(old_state) && 646 atomic_compare_exchange_strong_explicit(&mutex->state, &old_state, 0xffff, 647 memory_order_relaxed, memory_order_relaxed)) { 648 return 0; 649 } 650 return EBUSY; 651 } 652
