1 //===-- tsan_interface_atomic.cc ------------------------------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file is a part of ThreadSanitizer (TSan), a race detector. 11 // 12 //===----------------------------------------------------------------------===// 13 14 // ThreadSanitizer atomic operations are based on C++11/C1x standards. 15 // For background see C++11 standard. A slightly older, publically 16 // available draft of the standard (not entirely up-to-date, but close enough 17 // for casual browsing) is available here: 18 // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf 19 // The following page contains more background information: 20 // http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/ 21 22 #include "sanitizer_common/sanitizer_placement_new.h" 23 #include "sanitizer_common/sanitizer_stacktrace.h" 24 #include "tsan_interface_atomic.h" 25 #include "tsan_flags.h" 26 #include "tsan_rtl.h" 27 28 using namespace __tsan; // NOLINT 29 30 #define SCOPED_ATOMIC(func, ...) \ 31 const uptr callpc = (uptr)__builtin_return_address(0); \ 32 uptr pc = __sanitizer::StackTrace::GetCurrentPc(); \ 33 pc = __sanitizer::StackTrace::GetPreviousInstructionPc(pc); \ 34 mo = ConvertOrder(mo); \ 35 mo = flags()->force_seq_cst_atomics ? (morder)mo_seq_cst : mo; \ 36 ThreadState *const thr = cur_thread(); \ 37 AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \ 38 ScopedAtomic sa(thr, callpc, __FUNCTION__); \ 39 return Atomic##func(thr, pc, __VA_ARGS__); \ 40 /**/ 41 42 class ScopedAtomic { 43 public: 44 ScopedAtomic(ThreadState *thr, uptr pc, const char *func) 45 : thr_(thr) { 46 CHECK_EQ(thr_->in_rtl, 0); 47 ProcessPendingSignals(thr); 48 FuncEntry(thr_, pc); 49 DPrintf("#%d: %s\n", thr_->tid, func); 50 thr_->in_rtl++; 51 } 52 ~ScopedAtomic() { 53 thr_->in_rtl--; 54 CHECK_EQ(thr_->in_rtl, 0); 55 FuncExit(thr_); 56 } 57 private: 58 ThreadState *thr_; 59 }; 60 61 // Some shortcuts. 62 typedef __tsan_memory_order morder; 63 typedef __tsan_atomic8 a8; 64 typedef __tsan_atomic16 a16; 65 typedef __tsan_atomic32 a32; 66 typedef __tsan_atomic64 a64; 67 typedef __tsan_atomic128 a128; 68 const morder mo_relaxed = __tsan_memory_order_relaxed; 69 const morder mo_consume = __tsan_memory_order_consume; 70 const morder mo_acquire = __tsan_memory_order_acquire; 71 const morder mo_release = __tsan_memory_order_release; 72 const morder mo_acq_rel = __tsan_memory_order_acq_rel; 73 const morder mo_seq_cst = __tsan_memory_order_seq_cst; 74 75 static void AtomicStatInc(ThreadState *thr, uptr size, morder mo, StatType t) { 76 StatInc(thr, StatAtomic); 77 StatInc(thr, t); 78 StatInc(thr, size == 1 ? StatAtomic1 79 : size == 2 ? StatAtomic2 80 : size == 4 ? StatAtomic4 81 : size == 8 ? StatAtomic8 82 : StatAtomic16); 83 StatInc(thr, mo == mo_relaxed ? StatAtomicRelaxed 84 : mo == mo_consume ? StatAtomicConsume 85 : mo == mo_acquire ? StatAtomicAcquire 86 : mo == mo_release ? StatAtomicRelease 87 : mo == mo_acq_rel ? StatAtomicAcq_Rel 88 : StatAtomicSeq_Cst); 89 } 90 91 static bool IsLoadOrder(morder mo) { 92 return mo == mo_relaxed || mo == mo_consume 93 || mo == mo_acquire || mo == mo_seq_cst; 94 } 95 96 static bool IsStoreOrder(morder mo) { 97 return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst; 98 } 99 100 static bool IsReleaseOrder(morder mo) { 101 return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst; 102 } 103 104 static bool IsAcquireOrder(morder mo) { 105 return mo == mo_consume || mo == mo_acquire 106 || mo == mo_acq_rel || mo == mo_seq_cst; 107 } 108 109 static bool IsAcqRelOrder(morder mo) { 110 return mo == mo_acq_rel || mo == mo_seq_cst; 111 } 112 113 static morder ConvertOrder(morder mo) { 114 if (mo > (morder)100500) { 115 mo = morder(mo - 100500); 116 if (mo == morder(1 << 0)) 117 mo = mo_relaxed; 118 else if (mo == morder(1 << 1)) 119 mo = mo_consume; 120 else if (mo == morder(1 << 2)) 121 mo = mo_acquire; 122 else if (mo == morder(1 << 3)) 123 mo = mo_release; 124 else if (mo == morder(1 << 4)) 125 mo = mo_acq_rel; 126 else if (mo == morder(1 << 5)) 127 mo = mo_seq_cst; 128 } 129 CHECK_GE(mo, mo_relaxed); 130 CHECK_LE(mo, mo_seq_cst); 131 return mo; 132 } 133 134 template<typename T> T func_xchg(volatile T *v, T op) { 135 T res = __sync_lock_test_and_set(v, op); 136 // __sync_lock_test_and_set does not contain full barrier. 137 __sync_synchronize(); 138 return res; 139 } 140 141 template<typename T> T func_add(volatile T *v, T op) { 142 return __sync_fetch_and_add(v, op); 143 } 144 145 template<typename T> T func_sub(volatile T *v, T op) { 146 return __sync_fetch_and_sub(v, op); 147 } 148 149 template<typename T> T func_and(volatile T *v, T op) { 150 return __sync_fetch_and_and(v, op); 151 } 152 153 template<typename T> T func_or(volatile T *v, T op) { 154 return __sync_fetch_and_or(v, op); 155 } 156 157 template<typename T> T func_xor(volatile T *v, T op) { 158 return __sync_fetch_and_xor(v, op); 159 } 160 161 template<typename T> T func_nand(volatile T *v, T op) { 162 // clang does not support __sync_fetch_and_nand. 163 T cmp = *v; 164 for (;;) { 165 T newv = ~(cmp & op); 166 T cur = __sync_val_compare_and_swap(v, cmp, newv); 167 if (cmp == cur) 168 return cmp; 169 cmp = cur; 170 } 171 } 172 173 template<typename T> T func_cas(volatile T *v, T cmp, T xch) { 174 return __sync_val_compare_and_swap(v, cmp, xch); 175 } 176 177 // clang does not support 128-bit atomic ops. 178 // Atomic ops are executed under tsan internal mutex, 179 // here we assume that the atomic variables are not accessed 180 // from non-instrumented code. 181 #ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 182 a128 func_xchg(volatile a128 *v, a128 op) { 183 a128 cmp = *v; 184 *v = op; 185 return cmp; 186 } 187 188 a128 func_add(volatile a128 *v, a128 op) { 189 a128 cmp = *v; 190 *v = cmp + op; 191 return cmp; 192 } 193 194 a128 func_sub(volatile a128 *v, a128 op) { 195 a128 cmp = *v; 196 *v = cmp - op; 197 return cmp; 198 } 199 200 a128 func_and(volatile a128 *v, a128 op) { 201 a128 cmp = *v; 202 *v = cmp & op; 203 return cmp; 204 } 205 206 a128 func_or(volatile a128 *v, a128 op) { 207 a128 cmp = *v; 208 *v = cmp | op; 209 return cmp; 210 } 211 212 a128 func_xor(volatile a128 *v, a128 op) { 213 a128 cmp = *v; 214 *v = cmp ^ op; 215 return cmp; 216 } 217 218 a128 func_nand(volatile a128 *v, a128 op) { 219 a128 cmp = *v; 220 *v = ~(cmp & op); 221 return cmp; 222 } 223 224 a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) { 225 a128 cur = *v; 226 if (cur == cmp) 227 *v = xch; 228 return cur; 229 } 230 #endif 231 232 template<typename T> 233 static int SizeLog() { 234 if (sizeof(T) <= 1) 235 return kSizeLog1; 236 else if (sizeof(T) <= 2) 237 return kSizeLog2; 238 else if (sizeof(T) <= 4) 239 return kSizeLog4; 240 else 241 return kSizeLog8; 242 // For 16-byte atomics we also use 8-byte memory access, 243 // this leads to false negatives only in very obscure cases. 244 } 245 246 template<typename T> 247 static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, 248 morder mo) { 249 CHECK(IsLoadOrder(mo)); 250 // This fast-path is critical for performance. 251 // Assume the access is atomic. 252 if (!IsAcquireOrder(mo) && sizeof(T) <= sizeof(a)) { 253 MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>()); 254 return *a; 255 } 256 SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, false); 257 thr->clock.set(thr->tid, thr->fast_state.epoch()); 258 thr->clock.acquire(&s->clock); 259 T v = *a; 260 s->mtx.ReadUnlock(); 261 __sync_synchronize(); 262 MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>()); 263 return v; 264 } 265 266 template<typename T> 267 static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v, 268 morder mo) { 269 CHECK(IsStoreOrder(mo)); 270 MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>()); 271 // This fast-path is critical for performance. 272 // Assume the access is atomic. 273 // Strictly saying even relaxed store cuts off release sequence, 274 // so must reset the clock. 275 if (!IsReleaseOrder(mo) && sizeof(T) <= sizeof(a)) { 276 *a = v; 277 return; 278 } 279 __sync_synchronize(); 280 SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true); 281 thr->clock.set(thr->tid, thr->fast_state.epoch()); 282 thr->clock.ReleaseStore(&s->clock); 283 *a = v; 284 s->mtx.Unlock(); 285 // Trainling memory barrier to provide sequential consistency 286 // for Dekker-like store-load synchronization. 287 __sync_synchronize(); 288 } 289 290 template<typename T, T (*F)(volatile T *v, T op)> 291 static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { 292 MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>()); 293 SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true); 294 thr->clock.set(thr->tid, thr->fast_state.epoch()); 295 if (IsAcqRelOrder(mo)) 296 thr->clock.acq_rel(&s->clock); 297 else if (IsReleaseOrder(mo)) 298 thr->clock.release(&s->clock); 299 else if (IsAcquireOrder(mo)) 300 thr->clock.acquire(&s->clock); 301 v = F(a, v); 302 s->mtx.Unlock(); 303 return v; 304 } 305 306 template<typename T> 307 static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v, 308 morder mo) { 309 return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo); 310 } 311 312 template<typename T> 313 static T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v, 314 morder mo) { 315 return AtomicRMW<T, func_add>(thr, pc, a, v, mo); 316 } 317 318 template<typename T> 319 static T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v, 320 morder mo) { 321 return AtomicRMW<T, func_sub>(thr, pc, a, v, mo); 322 } 323 324 template<typename T> 325 static T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v, 326 morder mo) { 327 return AtomicRMW<T, func_and>(thr, pc, a, v, mo); 328 } 329 330 template<typename T> 331 static T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v, 332 morder mo) { 333 return AtomicRMW<T, func_or>(thr, pc, a, v, mo); 334 } 335 336 template<typename T> 337 static T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v, 338 morder mo) { 339 return AtomicRMW<T, func_xor>(thr, pc, a, v, mo); 340 } 341 342 template<typename T> 343 static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v, 344 morder mo) { 345 return AtomicRMW<T, func_nand>(thr, pc, a, v, mo); 346 } 347 348 template<typename T> 349 static bool AtomicCAS(ThreadState *thr, uptr pc, 350 volatile T *a, T *c, T v, morder mo, morder fmo) { 351 (void)fmo; // Unused because llvm does not pass it yet. 352 MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>()); 353 SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true); 354 thr->clock.set(thr->tid, thr->fast_state.epoch()); 355 if (IsAcqRelOrder(mo)) 356 thr->clock.acq_rel(&s->clock); 357 else if (IsReleaseOrder(mo)) 358 thr->clock.release(&s->clock); 359 else if (IsAcquireOrder(mo)) 360 thr->clock.acquire(&s->clock); 361 T cc = *c; 362 T pr = func_cas(a, cc, v); 363 s->mtx.Unlock(); 364 if (pr == cc) 365 return true; 366 *c = pr; 367 return false; 368 } 369 370 template<typename T> 371 static T AtomicCAS(ThreadState *thr, uptr pc, 372 volatile T *a, T c, T v, morder mo, morder fmo) { 373 AtomicCAS(thr, pc, a, &c, v, mo, fmo); 374 return c; 375 } 376 377 static void AtomicFence(ThreadState *thr, uptr pc, morder mo) { 378 // FIXME(dvyukov): not implemented. 379 __sync_synchronize(); 380 } 381 382 a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) { 383 SCOPED_ATOMIC(Load, a, mo); 384 } 385 386 a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) { 387 SCOPED_ATOMIC(Load, a, mo); 388 } 389 390 a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) { 391 SCOPED_ATOMIC(Load, a, mo); 392 } 393 394 a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) { 395 SCOPED_ATOMIC(Load, a, mo); 396 } 397 398 #if __TSAN_HAS_INT128 399 a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) { 400 SCOPED_ATOMIC(Load, a, mo); 401 } 402 #endif 403 404 void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) { 405 SCOPED_ATOMIC(Store, a, v, mo); 406 } 407 408 void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) { 409 SCOPED_ATOMIC(Store, a, v, mo); 410 } 411 412 void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) { 413 SCOPED_ATOMIC(Store, a, v, mo); 414 } 415 416 void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) { 417 SCOPED_ATOMIC(Store, a, v, mo); 418 } 419 420 #if __TSAN_HAS_INT128 421 void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) { 422 SCOPED_ATOMIC(Store, a, v, mo); 423 } 424 #endif 425 426 a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) { 427 SCOPED_ATOMIC(Exchange, a, v, mo); 428 } 429 430 a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) { 431 SCOPED_ATOMIC(Exchange, a, v, mo); 432 } 433 434 a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) { 435 SCOPED_ATOMIC(Exchange, a, v, mo); 436 } 437 438 a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) { 439 SCOPED_ATOMIC(Exchange, a, v, mo); 440 } 441 442 #if __TSAN_HAS_INT128 443 a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) { 444 SCOPED_ATOMIC(Exchange, a, v, mo); 445 } 446 #endif 447 448 a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) { 449 SCOPED_ATOMIC(FetchAdd, a, v, mo); 450 } 451 452 a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) { 453 SCOPED_ATOMIC(FetchAdd, a, v, mo); 454 } 455 456 a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) { 457 SCOPED_ATOMIC(FetchAdd, a, v, mo); 458 } 459 460 a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) { 461 SCOPED_ATOMIC(FetchAdd, a, v, mo); 462 } 463 464 #if __TSAN_HAS_INT128 465 a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) { 466 SCOPED_ATOMIC(FetchAdd, a, v, mo); 467 } 468 #endif 469 470 a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) { 471 SCOPED_ATOMIC(FetchSub, a, v, mo); 472 } 473 474 a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) { 475 SCOPED_ATOMIC(FetchSub, a, v, mo); 476 } 477 478 a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) { 479 SCOPED_ATOMIC(FetchSub, a, v, mo); 480 } 481 482 a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) { 483 SCOPED_ATOMIC(FetchSub, a, v, mo); 484 } 485 486 #if __TSAN_HAS_INT128 487 a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) { 488 SCOPED_ATOMIC(FetchSub, a, v, mo); 489 } 490 #endif 491 492 a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) { 493 SCOPED_ATOMIC(FetchAnd, a, v, mo); 494 } 495 496 a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) { 497 SCOPED_ATOMIC(FetchAnd, a, v, mo); 498 } 499 500 a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) { 501 SCOPED_ATOMIC(FetchAnd, a, v, mo); 502 } 503 504 a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) { 505 SCOPED_ATOMIC(FetchAnd, a, v, mo); 506 } 507 508 #if __TSAN_HAS_INT128 509 a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) { 510 SCOPED_ATOMIC(FetchAnd, a, v, mo); 511 } 512 #endif 513 514 a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) { 515 SCOPED_ATOMIC(FetchOr, a, v, mo); 516 } 517 518 a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) { 519 SCOPED_ATOMIC(FetchOr, a, v, mo); 520 } 521 522 a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) { 523 SCOPED_ATOMIC(FetchOr, a, v, mo); 524 } 525 526 a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) { 527 SCOPED_ATOMIC(FetchOr, a, v, mo); 528 } 529 530 #if __TSAN_HAS_INT128 531 a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) { 532 SCOPED_ATOMIC(FetchOr, a, v, mo); 533 } 534 #endif 535 536 a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) { 537 SCOPED_ATOMIC(FetchXor, a, v, mo); 538 } 539 540 a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) { 541 SCOPED_ATOMIC(FetchXor, a, v, mo); 542 } 543 544 a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) { 545 SCOPED_ATOMIC(FetchXor, a, v, mo); 546 } 547 548 a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) { 549 SCOPED_ATOMIC(FetchXor, a, v, mo); 550 } 551 552 #if __TSAN_HAS_INT128 553 a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) { 554 SCOPED_ATOMIC(FetchXor, a, v, mo); 555 } 556 #endif 557 558 a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) { 559 SCOPED_ATOMIC(FetchNand, a, v, mo); 560 } 561 562 a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) { 563 SCOPED_ATOMIC(FetchNand, a, v, mo); 564 } 565 566 a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) { 567 SCOPED_ATOMIC(FetchNand, a, v, mo); 568 } 569 570 a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) { 571 SCOPED_ATOMIC(FetchNand, a, v, mo); 572 } 573 574 #if __TSAN_HAS_INT128 575 a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) { 576 SCOPED_ATOMIC(FetchNand, a, v, mo); 577 } 578 #endif 579 580 int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v, 581 morder mo, morder fmo) { 582 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 583 } 584 585 int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v, 586 morder mo, morder fmo) { 587 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 588 } 589 590 int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v, 591 morder mo, morder fmo) { 592 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 593 } 594 595 int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v, 596 morder mo, morder fmo) { 597 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 598 } 599 600 #if __TSAN_HAS_INT128 601 int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v, 602 morder mo, morder fmo) { 603 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 604 } 605 #endif 606 607 int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, 608 morder mo, morder fmo) { 609 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 610 } 611 612 int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v, 613 morder mo, morder fmo) { 614 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 615 } 616 617 int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v, 618 morder mo, morder fmo) { 619 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 620 } 621 622 int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v, 623 morder mo, morder fmo) { 624 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 625 } 626 627 #if __TSAN_HAS_INT128 628 int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v, 629 morder mo, morder fmo) { 630 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 631 } 632 #endif 633 634 a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, 635 morder mo, morder fmo) { 636 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 637 } 638 a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, 639 morder mo, morder fmo) { 640 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 641 } 642 643 a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, 644 morder mo, morder fmo) { 645 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 646 } 647 648 a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, 649 morder mo, morder fmo) { 650 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 651 } 652 653 #if __TSAN_HAS_INT128 654 a128 __tsan_atomic64_compare_exchange_val(volatile a128 *a, a128 c, a128 v, 655 morder mo, morder fmo) { 656 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 657 } 658 #endif 659 660 void __tsan_atomic_thread_fence(morder mo) { 661 char* a; 662 SCOPED_ATOMIC(Fence, mo); 663 } 664 665 void __tsan_atomic_signal_fence(morder mo) { 666 } 667