Lines Matching refs:mo
59 static bool IsLoadOrder(morder mo) {
60 return mo == mo_relaxed || mo == mo_consume
61 || mo == mo_acquire || mo == mo_seq_cst;
64 static bool IsStoreOrder(morder mo) {
65 return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst;
68 static bool IsReleaseOrder(morder mo) {
69 return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst;
72 static bool IsAcquireOrder(morder mo) {
73 return mo == mo_consume || mo == mo_acquire
74 || mo == mo_acq_rel || mo == mo_seq_cst;
77 static bool IsAcqRelOrder(morder mo) {
78 return mo == mo_acq_rel || mo == mo_seq_cst;
220 static memory_order to_mo(morder mo) {
221 switch (mo) {
234 static T NoTsanAtomicLoad(const volatile T *a, morder mo) {
235 return atomic_load(to_atomic(a), to_mo(mo));
239 static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) {
247 morder mo) {
248 CHECK(IsLoadOrder(mo));
251 if (!IsAcquireOrder(mo)) {
253 return NoTsanAtomicLoad(a, mo);
257 T v = NoTsanAtomicLoad(a, mo);
264 static void NoTsanAtomicStore(volatile T *a, T v, morder mo) {
265 atomic_store(to_atomic(a), v, to_mo(mo));
269 static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) {
277 morder mo) {
278 CHECK(IsStoreOrder(mo));
284 if (!IsReleaseOrder(mo)) {
285 NoTsanAtomicStore(a, v, mo);
294 NoTsanAtomicStore(a, v, mo);
299 static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) {
302 if (mo != mo_relaxed) {
307 if (IsAcqRelOrder(mo))
309 else if (IsReleaseOrder(mo))
311 else if (IsAcquireOrder(mo))
321 static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) {
326 static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) {
331 static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) {
336 static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) {
341 static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) {
346 static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) {
351 static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) {
357 morder mo) {
358 return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo);
363 morder mo) {
364 return AtomicRMW<T, func_add>(thr, pc, a, v, mo);
369 morder mo) {
370 return AtomicRMW<T, func_sub>(thr, pc, a, v, mo);
375 morder mo) {
376 return AtomicRMW<T, func_and>(thr, pc, a, v, mo);
381 morder mo) {
382 return AtomicRMW<T, func_or>(thr, pc, a, v, mo);
387 morder mo) {
388 return AtomicRMW<T, func_xor>(thr, pc, a, v, mo);
393 morder mo) {
394 return AtomicRMW<T, func_nand>(thr, pc, a, v, mo);
398 static bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) {
399 return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo));
404 morder mo, morder fmo) {
415 static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) {
416 NoTsanAtomicCAS(a, &c, v, mo, fmo);
422 volatile T *a, T *c, T v, morder mo, morder fmo) {
426 bool write_lock = mo != mo_acquire && mo != mo_consume;
427 if (mo != mo_relaxed) {
432 if (IsAcqRelOrder(mo))
434 else if (IsReleaseOrder(mo))
436 else if (IsAcquireOrder(mo))
455 volatile T *a, T c, T v, morder mo, morder fmo) {
456 AtomicCAS(thr, pc, a, &c, v, mo, fmo);
461 static void NoTsanAtomicFence(morder mo) {
465 static void AtomicFence(ThreadState *thr, uptr pc, morder mo) {
479 mo = flags()->force_seq_cst_atomics ? (morder)mo_seq_cst : mo; \
483 AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \
484 ScopedAtomic sa(thr, callpc, a, mo, __func__); \
491 morder mo, const char *func)
494 DPrintf("#%d: %s(%p, %d)\n", thr_->tid, func, a, mo);
504 static void AtomicStatInc(ThreadState *thr, uptr size, morder mo, StatType t) {
512 StatInc(thr, mo == mo_relaxed ? StatAtomicRelaxed
513 : mo == mo_consume ? StatAtomicConsume
514 : mo == mo_acquire ? StatAtomicAcquire
515 : mo == mo_release ? StatAtomicRelease
516 : mo == mo_acq_rel ? StatAtomicAcq_Rel
522 a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) {
523 SCOPED_ATOMIC(Load, a, mo);
527 a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) {
528 SCOPED_ATOMIC(Load, a, mo);
532 a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) {
533 SCOPED_ATOMIC(Load, a, mo);
537 a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) {
538 SCOPED_ATOMIC(Load, a, mo);
543 a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) {
544 SCOPED_ATOMIC(Load, a, mo);
549 void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) {
550 SCOPED_ATOMIC(Store, a, v, mo);
554 void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) {
555 SCOPED_ATOMIC(Store, a, v, mo);
559 void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) {
560 SCOPED_ATOMIC(Store, a, v, mo);
564 void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) {
565 SCOPED_ATOMIC(Store, a, v, mo);
570 void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) {
571 SCOPED_ATOMIC(Store, a, v, mo);
576 a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) {
577 SCOPED_ATOMIC(Exchange, a, v, mo);
581 a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) {
582 SCOPED_ATOMIC(Exchange, a, v, mo);
586 a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) {
587 SCOPED_ATOMIC(Exchange, a, v, mo);
591 a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) {
592 SCOPED_ATOMIC(Exchange, a, v, mo);
597 mo) {
598 SCOPED_ATOMIC(Exchange, a, v, mo);
603 a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) {
604 SCOPED_ATOMIC(FetchAdd, a, v, mo);
608 a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) {
609 SCOPED_ATOMIC(FetchAdd, a, v, mo);
613 a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) {
614 SCOPED_ATOMIC(FetchAdd, a, v, mo);
618 a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) {
619 SCOPED_ATOMIC(FetchAdd, a, v, mo);
624 a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) {
625 SCOPED_ATOMIC(FetchAdd, a, v, mo);
630 a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) {
631 SCOPED_ATOMIC(FetchSub, a, v, mo);
635 a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) {
636 SCOPED_ATOMIC(FetchSub, a, v, mo);
640 a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) {
641 SCOPED_ATOMIC(FetchSub, a, v, mo);
645 a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) {
646 SCOPED_ATOMIC(FetchSub, a, v, mo);
651 a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) {
652 SCOPED_ATOMIC(FetchSub, a, v, mo);
657 a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) {
658 SCOPED_ATOMIC(FetchAnd, a, v, mo);
662 a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) {
663 SCOPED_ATOMIC(FetchAnd, a, v, mo);
667 a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) {
668 SCOPED_ATOMIC(FetchAnd, a, v, mo);
672 a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) {
673 SCOPED_ATOMIC(FetchAnd, a, v, mo);
678 a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) {
679 SCOPED_ATOMIC(FetchAnd, a, v, mo);
684 a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) {
685 SCOPED_ATOMIC(FetchOr, a, v, mo);
689 a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) {
690 SCOPED_ATOMIC(FetchOr, a, v, mo);
694 a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) {
695 SCOPED_ATOMIC(FetchOr, a, v, mo);
699 a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) {
700 SCOPED_ATOMIC(FetchOr, a, v, mo);
705 a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) {
706 SCOPED_ATOMIC(FetchOr, a, v, mo);
711 a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) {
712 SCOPED_ATOMIC(FetchXor, a, v, mo);
716 a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) {
717 SCOPED_ATOMIC(FetchXor, a, v, mo);
721 a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) {
722 SCOPED_ATOMIC(FetchXor, a, v, mo);
726 a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) {
727 SCOPED_ATOMIC(FetchXor, a, v, mo);
732 a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) {
733 SCOPED_ATOMIC(FetchXor, a, v, mo);
738 a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) {
739 SCOPED_ATOMIC(FetchNand, a, v, mo);
743 a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) {
744 SCOPED_ATOMIC(FetchNand, a, v, mo);
748 a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) {
749 SCOPED_ATOMIC(FetchNand, a, v, mo);
753 a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) {
754 SCOPED_ATOMIC(FetchNand, a, v, mo);
759 a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) {
760 SCOPED_ATOMIC(FetchNand, a, v, mo);
766 morder mo, morder fmo) {
767 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
772 morder mo, morder fmo) {
773 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
778 morder mo, morder fmo) {
779 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
784 morder mo, morder fmo) {
785 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
791 morder mo, morder fmo) {
792 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
798 morder mo, morder fmo) {
799 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
804 morder mo, morder fmo) {
805 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
810 morder mo, morder fmo) {
811 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
816 morder mo, morder fmo) {
817 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
823 morder mo, morder fmo) {
824 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
830 morder mo, morder fmo) {
831 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
836 morder mo, morder fmo) {
837 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
842 morder mo, morder fmo) {
843 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
848 morder mo, morder fmo) {
849 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
855 morder mo, morder fmo) {
856 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
861 void __tsan_atomic_thread_fence(morder mo) {
863 SCOPED_ATOMIC(Fence, mo);
867 void __tsan_atomic_signal_fence(morder mo) {
