Home | History | Annotate | Download | only in include 
      1 // -*- C++ -*-
      2 //===--------------------------- atomic -----------------------------------===//
      3 //
      4 //                     The LLVM Compiler Infrastructure
      5 //
      6 // This file is distributed under the University of Illinois Open Source
      7 // License. See LICENSE.TXT for details.
      8 //
      9 //===----------------------------------------------------------------------===//
     10 
     11 #ifndef _LIBCPP_ATOMIC
     12 #define _LIBCPP_ATOMIC
     13 
     14 /*
     15     atomic synopsis
     16 
     17 namespace std
     18 {
     19 
     20 // order and consistency
     21 
     22 typedef enum memory_order
     23 {
     24     memory_order_relaxed,
     25     memory_order_consume,  // load-consume
     26     memory_order_acquire,  // load-acquire
     27     memory_order_release,  // store-release
     28     memory_order_acq_rel,  // store-release load-acquire
     29     memory_order_seq_cst   // store-release load-acquire
     30 } memory_order;
     31 
     32 template <class T> T kill_dependency(T y) noexcept;
     33 
     34 // lock-free property
     35 
     36 #define ATOMIC_BOOL_LOCK_FREE unspecified
     37 #define ATOMIC_CHAR_LOCK_FREE unspecified
     38 #define ATOMIC_CHAR16_T_LOCK_FREE unspecified
     39 #define ATOMIC_CHAR32_T_LOCK_FREE unspecified
     40 #define ATOMIC_WCHAR_T_LOCK_FREE unspecified
     41 #define ATOMIC_SHORT_LOCK_FREE unspecified
     42 #define ATOMIC_INT_LOCK_FREE unspecified
     43 #define ATOMIC_LONG_LOCK_FREE unspecified
     44 #define ATOMIC_LLONG_LOCK_FREE unspecified
     45 #define ATOMIC_POINTER_LOCK_FREE unspecified
     46 
     47 // flag type and operations
     48 
     49 typedef struct atomic_flag
     50 {
     51     bool test_and_set(memory_order m = memory_order_seq_cst) volatile noexcept;
     52     bool test_and_set(memory_order m = memory_order_seq_cst) noexcept;
     53     void clear(memory_order m = memory_order_seq_cst) volatile noexcept;
     54     void clear(memory_order m = memory_order_seq_cst) noexcept;
     55     atomic_flag()  noexcept = default;
     56     atomic_flag(const atomic_flag&) = delete;
     57     atomic_flag& operator=(const atomic_flag&) = delete;
     58     atomic_flag& operator=(const atomic_flag&) volatile = delete;
     59 } atomic_flag;
     60 
     61 bool
     62     atomic_flag_test_and_set(volatile atomic_flag* obj) noexcept;
     63 
     64 bool
     65     atomic_flag_test_and_set(atomic_flag* obj) noexcept;
     66 
     67 bool
     68     atomic_flag_test_and_set_explicit(volatile atomic_flag* obj,
     69                                       memory_order m) noexcept;
     70 
     71 bool
     72     atomic_flag_test_and_set_explicit(atomic_flag* obj, memory_order m) noexcept;
     73 
     74 void
     75     atomic_flag_clear(volatile atomic_flag* obj) noexcept;
     76 
     77 void
     78     atomic_flag_clear(atomic_flag* obj) noexcept;
     79 
     80 void
     81     atomic_flag_clear_explicit(volatile atomic_flag* obj, memory_order m) noexcept;
     82 
     83 void
     84     atomic_flag_clear_explicit(atomic_flag* obj, memory_order m) noexcept;
     85 
     86 #define ATOMIC_FLAG_INIT see below
     87 #define ATOMIC_VAR_INIT(value) see below
     88 
     89 template <class T>
     90 struct atomic
     91 {
     92     bool is_lock_free() const volatile noexcept;
     93     bool is_lock_free() const noexcept;
     94     void store(T desr, memory_order m = memory_order_seq_cst) volatile noexcept;
     95     void store(T desr, memory_order m = memory_order_seq_cst) noexcept;
     96     T load(memory_order m = memory_order_seq_cst) const volatile noexcept;
     97     T load(memory_order m = memory_order_seq_cst) const noexcept;
     98     operator T() const volatile noexcept;
     99     operator T() const noexcept;
    100     T exchange(T desr, memory_order m = memory_order_seq_cst) volatile noexcept;
    101     T exchange(T desr, memory_order m = memory_order_seq_cst) noexcept;
    102     bool compare_exchange_weak(T& expc, T desr,
    103                                memory_order s, memory_order f) volatile noexcept;
    104     bool compare_exchange_weak(T& expc, T desr, memory_order s, memory_order f) noexcept;
    105     bool compare_exchange_strong(T& expc, T desr,
    106                                  memory_order s, memory_order f) volatile noexcept;
    107     bool compare_exchange_strong(T& expc, T desr,
    108                                  memory_order s, memory_order f) noexcept;
    109     bool compare_exchange_weak(T& expc, T desr,
    110                                memory_order m = memory_order_seq_cst) volatile noexcept;
    111     bool compare_exchange_weak(T& expc, T desr,
    112                                memory_order m = memory_order_seq_cst) noexcept;
    113     bool compare_exchange_strong(T& expc, T desr,
    114                                 memory_order m = memory_order_seq_cst) volatile noexcept;
    115     bool compare_exchange_strong(T& expc, T desr,
    116                                  memory_order m = memory_order_seq_cst) noexcept;
    117 
    118     atomic() noexcept = default;
    119     constexpr atomic(T desr) noexcept;
    120     atomic(const atomic&) = delete;
    121     atomic& operator=(const atomic&) = delete;
    122     atomic& operator=(const atomic&) volatile = delete;
    123     T operator=(T) volatile noexcept;
    124     T operator=(T) noexcept;
    125 };
    126 
    127 template <>
    128 struct atomic<integral>
    129 {
    130     bool is_lock_free() const volatile noexcept;
    131     bool is_lock_free() const noexcept;
    132     void store(integral desr, memory_order m = memory_order_seq_cst) volatile noexcept;
    133     void store(integral desr, memory_order m = memory_order_seq_cst) noexcept;
    134     integral load(memory_order m = memory_order_seq_cst) const volatile noexcept;
    135     integral load(memory_order m = memory_order_seq_cst) const noexcept;
    136     operator integral() const volatile noexcept;
    137     operator integral() const noexcept;
    138     integral exchange(integral desr,
    139                       memory_order m = memory_order_seq_cst) volatile noexcept;
    140     integral exchange(integral desr, memory_order m = memory_order_seq_cst) noexcept;
    141     bool compare_exchange_weak(integral& expc, integral desr,
    142                                memory_order s, memory_order f) volatile noexcept;
    143     bool compare_exchange_weak(integral& expc, integral desr,
    144                                memory_order s, memory_order f) noexcept;
    145     bool compare_exchange_strong(integral& expc, integral desr,
    146                                  memory_order s, memory_order f) volatile noexcept;
    147     bool compare_exchange_strong(integral& expc, integral desr,
    148                                  memory_order s, memory_order f) noexcept;
    149     bool compare_exchange_weak(integral& expc, integral desr,
    150                                memory_order m = memory_order_seq_cst) volatile noexcept;
    151     bool compare_exchange_weak(integral& expc, integral desr,
    152                                memory_order m = memory_order_seq_cst) noexcept;
    153     bool compare_exchange_strong(integral& expc, integral desr,
    154                                 memory_order m = memory_order_seq_cst) volatile noexcept;
    155     bool compare_exchange_strong(integral& expc, integral desr,
    156                                  memory_order m = memory_order_seq_cst) noexcept;
    157 
    158     integral
    159         fetch_add(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
    160     integral fetch_add(integral op, memory_order m = memory_order_seq_cst) noexcept;
    161     integral
    162         fetch_sub(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
    163     integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) noexcept;
    164     integral
    165         fetch_and(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
    166     integral fetch_and(integral op, memory_order m = memory_order_seq_cst) noexcept;
    167     integral
    168         fetch_or(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
    169     integral fetch_or(integral op, memory_order m = memory_order_seq_cst) noexcept;
    170     integral
    171         fetch_xor(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
    172     integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) noexcept;
    173 
    174     atomic() noexcept = default;
    175     constexpr atomic(integral desr) noexcept;
    176     atomic(const atomic&) = delete;
    177     atomic& operator=(const atomic&) = delete;
    178     atomic& operator=(const atomic&) volatile = delete;
    179     integral operator=(integral desr) volatile noexcept;
    180     integral operator=(integral desr) noexcept;
    181 
    182     integral operator++(int) volatile noexcept;
    183     integral operator++(int) noexcept;
    184     integral operator--(int) volatile noexcept;
    185     integral operator--(int) noexcept;
    186     integral operator++() volatile noexcept;
    187     integral operator++() noexcept;
    188     integral operator--() volatile noexcept;
    189     integral operator--() noexcept;
    190     integral operator+=(integral op) volatile noexcept;
    191     integral operator+=(integral op) noexcept;
    192     integral operator-=(integral op) volatile noexcept;
    193     integral operator-=(integral op) noexcept;
    194     integral operator&=(integral op) volatile noexcept;
    195     integral operator&=(integral op) noexcept;
    196     integral operator|=(integral op) volatile noexcept;
    197     integral operator|=(integral op) noexcept;
    198     integral operator^=(integral op) volatile noexcept;
    199     integral operator^=(integral op) noexcept;
    200 };
    201 
    202 template <class T>
    203 struct atomic<T*>
    204 {
    205     bool is_lock_free() const volatile noexcept;
    206     bool is_lock_free() const noexcept;
    207     void store(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept;
    208     void store(T* desr, memory_order m = memory_order_seq_cst) noexcept;
    209     T* load(memory_order m = memory_order_seq_cst) const volatile noexcept;
    210     T* load(memory_order m = memory_order_seq_cst) const noexcept;
    211     operator T*() const volatile noexcept;
    212     operator T*() const noexcept;
    213     T* exchange(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept;
    214     T* exchange(T* desr, memory_order m = memory_order_seq_cst) noexcept;
    215     bool compare_exchange_weak(T*& expc, T* desr,
    216                                memory_order s, memory_order f) volatile noexcept;
    217     bool compare_exchange_weak(T*& expc, T* desr,
    218                                memory_order s, memory_order f) noexcept;
    219     bool compare_exchange_strong(T*& expc, T* desr,
    220                                  memory_order s, memory_order f) volatile noexcept;
    221     bool compare_exchange_strong(T*& expc, T* desr,
    222                                  memory_order s, memory_order f) noexcept;
    223     bool compare_exchange_weak(T*& expc, T* desr,
    224                                memory_order m = memory_order_seq_cst) volatile noexcept;
    225     bool compare_exchange_weak(T*& expc, T* desr,
    226                                memory_order m = memory_order_seq_cst) noexcept;
    227     bool compare_exchange_strong(T*& expc, T* desr,
    228                                 memory_order m = memory_order_seq_cst) volatile noexcept;
    229     bool compare_exchange_strong(T*& expc, T* desr,
    230                                  memory_order m = memory_order_seq_cst) noexcept;
    231     T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept;
    232     T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept;
    233     T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept;
    234     T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept;
    235 
    236     atomic() noexcept = default;
    237     constexpr atomic(T* desr) noexcept;
    238     atomic(const atomic&) = delete;
    239     atomic& operator=(const atomic&) = delete;
    240     atomic& operator=(const atomic&) volatile = delete;
    241 
    242     T* operator=(T*) volatile noexcept;
    243     T* operator=(T*) noexcept;
    244     T* operator++(int) volatile noexcept;
    245     T* operator++(int) noexcept;
    246     T* operator--(int) volatile noexcept;
    247     T* operator--(int) noexcept;
    248     T* operator++() volatile noexcept;
    249     T* operator++() noexcept;
    250     T* operator--() volatile noexcept;
    251     T* operator--() noexcept;
    252     T* operator+=(ptrdiff_t op) volatile noexcept;
    253     T* operator+=(ptrdiff_t op) noexcept;
    254     T* operator-=(ptrdiff_t op) volatile noexcept;
    255     T* operator-=(ptrdiff_t op) noexcept;
    256 };
    257 
    258 
    259 template <class T>
    260     bool
    261     atomic_is_lock_free(const volatile atomic<T>* obj) noexcept;
    262 
    263 template <class T>
    264     bool
    265     atomic_is_lock_free(const atomic<T>* obj) noexcept;
    266 
    267 template <class T>
    268     void
    269     atomic_init(volatile atomic<T>* obj, T desr) noexcept;
    270 
    271 template <class T>
    272     void
    273     atomic_init(atomic<T>* obj, T desr) noexcept;
    274 
    275 template <class T>
    276     void
    277     atomic_store(volatile atomic<T>* obj, T desr) noexcept;
    278 
    279 template <class T>
    280     void
    281     atomic_store(atomic<T>* obj, T desr) noexcept;
    282 
    283 template <class T>
    284     void
    285     atomic_store_explicit(volatile atomic<T>* obj, T desr, memory_order m) noexcept;
    286 
    287 template <class T>
    288     void
    289     atomic_store_explicit(atomic<T>* obj, T desr, memory_order m) noexcept;
    290 
    291 template <class T>
    292     T
    293     atomic_load(const volatile atomic<T>* obj) noexcept;
    294 
    295 template <class T>
    296     T
    297     atomic_load(const atomic<T>* obj) noexcept;
    298 
    299 template <class T>
    300     T
    301     atomic_load_explicit(const volatile atomic<T>* obj, memory_order m) noexcept;
    302 
    303 template <class T>
    304     T
    305     atomic_load_explicit(const atomic<T>* obj, memory_order m) noexcept;
    306 
    307 template <class T>
    308     T
    309     atomic_exchange(volatile atomic<T>* obj, T desr) noexcept;
    310 
    311 template <class T>
    312     T
    313     atomic_exchange(atomic<T>* obj, T desr) noexcept;
    314 
    315 template <class T>
    316     T
    317     atomic_exchange_explicit(volatile atomic<T>* obj, T desr, memory_order m) noexcept;
    318 
    319 template <class T>
    320     T
    321     atomic_exchange_explicit(atomic<T>* obj, T desr, memory_order m) noexcept;
    322 
    323 template <class T>
    324     bool
    325     atomic_compare_exchange_weak(volatile atomic<T>* obj, T* expc, T desr) noexcept;
    326 
    327 template <class T>
    328     bool
    329     atomic_compare_exchange_weak(atomic<T>* obj, T* expc, T desr) noexcept;
    330 
    331 template <class T>
    332     bool
    333     atomic_compare_exchange_strong(volatile atomic<T>* obj, T* expc, T desr) noexcept;
    334 
    335 template <class T>
    336     bool
    337     atomic_compare_exchange_strong(atomic<T>* obj, T* expc, T desr) noexcept;
    338 
    339 template <class T>
    340     bool
    341     atomic_compare_exchange_weak_explicit(volatile atomic<T>* obj, T* expc,
    342                                           T desr,
    343                                           memory_order s, memory_order f) noexcept;
    344 
    345 template <class T>
    346     bool
    347     atomic_compare_exchange_weak_explicit(atomic<T>* obj, T* expc, T desr,
    348                                           memory_order s, memory_order f) noexcept;
    349 
    350 template <class T>
    351     bool
    352     atomic_compare_exchange_strong_explicit(volatile atomic<T>* obj,
    353                                             T* expc, T desr,
    354                                             memory_order s, memory_order f) noexcept;
    355 
    356 template <class T>
    357     bool
    358     atomic_compare_exchange_strong_explicit(atomic<T>* obj, T* expc,
    359                                             T desr,
    360                                             memory_order s, memory_order f) noexcept;
    361 
    362 template <class Integral>
    363     Integral
    364     atomic_fetch_add(volatile atomic<Integral>* obj, Integral op) noexcept;
    365 
    366 template <class Integral>
    367     Integral
    368     atomic_fetch_add(atomic<Integral>* obj, Integral op) noexcept;
    369 
    370 template <class Integral>
    371     Integral
    372     atomic_fetch_add_explicit(volatile atomic<Integral>* obj, Integral op,
    373                               memory_order m) noexcept;
    374 template <class Integral>
    375     Integral
    376     atomic_fetch_add_explicit(atomic<Integral>* obj, Integral op,
    377                               memory_order m) noexcept;
    378 template <class Integral>
    379     Integral
    380     atomic_fetch_sub(volatile atomic<Integral>* obj, Integral op) noexcept;
    381 
    382 template <class Integral>
    383     Integral
    384     atomic_fetch_sub(atomic<Integral>* obj, Integral op) noexcept;
    385 
    386 template <class Integral>
    387     Integral
    388     atomic_fetch_sub_explicit(volatile atomic<Integral>* obj, Integral op,
    389                               memory_order m) noexcept;
    390 template <class Integral>
    391     Integral
    392     atomic_fetch_sub_explicit(atomic<Integral>* obj, Integral op,
    393                               memory_order m) noexcept;
    394 template <class Integral>
    395     Integral
    396     atomic_fetch_and(volatile atomic<Integral>* obj, Integral op) noexcept;
    397 
    398 template <class Integral>
    399     Integral
    400     atomic_fetch_and(atomic<Integral>* obj, Integral op) noexcept;
    401 
    402 template <class Integral>
    403     Integral
    404     atomic_fetch_and_explicit(volatile atomic<Integral>* obj, Integral op,
    405                               memory_order m) noexcept;
    406 template <class Integral>
    407     Integral
    408     atomic_fetch_and_explicit(atomic<Integral>* obj, Integral op,
    409                               memory_order m) noexcept;
    410 template <class Integral>
    411     Integral
    412     atomic_fetch_or(volatile atomic<Integral>* obj, Integral op) noexcept;
    413 
    414 template <class Integral>
    415     Integral
    416     atomic_fetch_or(atomic<Integral>* obj, Integral op) noexcept;
    417 
    418 template <class Integral>
    419     Integral
    420     atomic_fetch_or_explicit(volatile atomic<Integral>* obj, Integral op,
    421                              memory_order m) noexcept;
    422 template <class Integral>
    423     Integral
    424     atomic_fetch_or_explicit(atomic<Integral>* obj, Integral op,
    425                              memory_order m) noexcept;
    426 template <class Integral>
    427     Integral
    428     atomic_fetch_xor(volatile atomic<Integral>* obj, Integral op) noexcept;
    429 
    430 template <class Integral>
    431     Integral
    432     atomic_fetch_xor(atomic<Integral>* obj, Integral op) noexcept;
    433 
    434 template <class Integral>
    435     Integral
    436     atomic_fetch_xor_explicit(volatile atomic<Integral>* obj, Integral op,
    437                               memory_order m) noexcept;
    438 template <class Integral>
    439     Integral
    440     atomic_fetch_xor_explicit(atomic<Integral>* obj, Integral op,
    441                               memory_order m) noexcept;
    442 
    443 template <class T>
    444     T*
    445     atomic_fetch_add(volatile atomic<T*>* obj, ptrdiff_t op) noexcept;
    446 
    447 template <class T>
    448     T*
    449     atomic_fetch_add(atomic<T*>* obj, ptrdiff_t op) noexcept;
    450 
    451 template <class T>
    452     T*
    453     atomic_fetch_add_explicit(volatile atomic<T*>* obj, ptrdiff_t op,
    454                               memory_order m) noexcept;
    455 template <class T>
    456     T*
    457     atomic_fetch_add_explicit(atomic<T*>* obj, ptrdiff_t op, memory_order m) noexcept;
    458 
    459 template <class T>
    460     T*
    461     atomic_fetch_sub(volatile atomic<T*>* obj, ptrdiff_t op) noexcept;
    462 
    463 template <class T>
    464     T*
    465     atomic_fetch_sub(atomic<T*>* obj, ptrdiff_t op) noexcept;
    466 
    467 template <class T>
    468     T*
    469     atomic_fetch_sub_explicit(volatile atomic<T*>* obj, ptrdiff_t op,
    470                               memory_order m) noexcept;
    471 template <class T>
    472     T*
    473     atomic_fetch_sub_explicit(atomic<T*>* obj, ptrdiff_t op, memory_order m) noexcept;
    474 
    475 // Atomics for standard typedef types
    476 
    477 typedef atomic<bool>               atomic_bool;
    478 typedef atomic<char>               atomic_char;
    479 typedef atomic<signed char>        atomic_schar;
    480 typedef atomic<unsigned char>      atomic_uchar;
    481 typedef atomic<short>              atomic_short;
    482 typedef atomic<unsigned short>     atomic_ushort;
    483 typedef atomic<int>                atomic_int;
    484 typedef atomic<unsigned int>       atomic_uint;
    485 typedef atomic<long>               atomic_long;
    486 typedef atomic<unsigned long>      atomic_ulong;
    487 typedef atomic<long long>          atomic_llong;
    488 typedef atomic<unsigned long long> atomic_ullong;
    489 typedef atomic<char16_t>           atomic_char16_t;
    490 typedef atomic<char32_t>           atomic_char32_t;
    491 typedef atomic<wchar_t>            atomic_wchar_t;
    492 
    493 typedef atomic<int_least8_t>   atomic_int_least8_t;
    494 typedef atomic<uint_least8_t>  atomic_uint_least8_t;
    495 typedef atomic<int_least16_t>  atomic_int_least16_t;
    496 typedef atomic<uint_least16_t> atomic_uint_least16_t;
    497 typedef atomic<int_least32_t>  atomic_int_least32_t;
    498 typedef atomic<uint_least32_t> atomic_uint_least32_t;
    499 typedef atomic<int_least64_t>  atomic_int_least64_t;
    500 typedef atomic<uint_least64_t> atomic_uint_least64_t;
    501 
    502 typedef atomic<int_fast8_t>   atomic_int_fast8_t;
    503 typedef atomic<uint_fast8_t>  atomic_uint_fast8_t;
    504 typedef atomic<int_fast16_t>  atomic_int_fast16_t;
    505 typedef atomic<uint_fast16_t> atomic_uint_fast16_t;
    506 typedef atomic<int_fast32_t>  atomic_int_fast32_t;
    507 typedef atomic<uint_fast32_t> atomic_uint_fast32_t;
    508 typedef atomic<int_fast64_t>  atomic_int_fast64_t;
    509 typedef atomic<uint_fast64_t> atomic_uint_fast64_t;
    510 
    511 typedef atomic<intptr_t>  atomic_intptr_t;
    512 typedef atomic<uintptr_t> atomic_uintptr_t;
    513 typedef atomic<size_t>    atomic_size_t;
    514 typedef atomic<ptrdiff_t> atomic_ptrdiff_t;
    515 typedef atomic<intmax_t>  atomic_intmax_t;
    516 typedef atomic<uintmax_t> atomic_uintmax_t;
    517 
    518 // fences
    519 
    520 void atomic_thread_fence(memory_order m) noexcept;
    521 void atomic_signal_fence(memory_order m) noexcept;
    522 
    523 }  // std
    524 
    525 */
    526 
    527 #include <__config>
    528 #include <cstddef>
    529 #include <cstdint>
    530 #include <type_traits>
    531 
    532 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
    533 #pragma GCC system_header
    534 #endif
    535 
    536 _LIBCPP_BEGIN_NAMESPACE_STD
    537 
    538 #if !__has_feature(cxx_atomic)
    539 #error <atomic> is not implemented
    540 #else
    541 
    542 typedef enum memory_order
    543 {
    544     memory_order_relaxed, memory_order_consume, memory_order_acquire,
    545     memory_order_release, memory_order_acq_rel, memory_order_seq_cst
    546 } memory_order;
    547 
    548 template <class _Tp>
    549 inline _LIBCPP_INLINE_VISIBILITY
    550 _Tp
    551 kill_dependency(_Tp __y) _NOEXCEPT
    552 {
    553     return __y;
    554 }
    555 
    556 // general atomic<T>
    557 
    558 template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value>
    559 struct __atomic_base  // false
    560 {
    561     mutable _Atomic(_Tp) __a_;
    562 
    563     _LIBCPP_INLINE_VISIBILITY
    564     bool is_lock_free() const volatile _NOEXCEPT
    565         {return __c11_atomic_is_lock_free(sizeof(_Tp));}
    566     _LIBCPP_INLINE_VISIBILITY
    567     bool is_lock_free() const _NOEXCEPT
    568         {return __c11_atomic_is_lock_free(sizeof(_Tp));}
    569     _LIBCPP_INLINE_VISIBILITY
    570     void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
    571         {__c11_atomic_store(&__a_, __d, __m);}
    572     _LIBCPP_INLINE_VISIBILITY
    573     void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT
    574         {__c11_atomic_store(&__a_, __d, __m);}
    575     _LIBCPP_INLINE_VISIBILITY
    576     _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
    577         {return __c11_atomic_load(&__a_, __m);}
    578     _LIBCPP_INLINE_VISIBILITY
    579     _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT
    580         {return __c11_atomic_load(&__a_, __m);}
    581     _LIBCPP_INLINE_VISIBILITY
    582     operator _Tp() const volatile _NOEXCEPT {return load();}
    583     _LIBCPP_INLINE_VISIBILITY
    584     operator _Tp() const _NOEXCEPT          {return load();}
    585     _LIBCPP_INLINE_VISIBILITY
    586     _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
    587         {return __c11_atomic_exchange(&__a_, __d, __m);}
    588     _LIBCPP_INLINE_VISIBILITY
    589     _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT
    590         {return __c11_atomic_exchange(&__a_, __d, __m);}
    591     _LIBCPP_INLINE_VISIBILITY
    592     bool compare_exchange_weak(_Tp& __e, _Tp __d,
    593                                memory_order __s, memory_order __f) volatile _NOEXCEPT
    594         {return __c11_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);}
    595     _LIBCPP_INLINE_VISIBILITY
    596     bool compare_exchange_weak(_Tp& __e, _Tp __d,
    597                                memory_order __s, memory_order __f) _NOEXCEPT
    598         {return __c11_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);}
    599     _LIBCPP_INLINE_VISIBILITY
    600     bool compare_exchange_strong(_Tp& __e, _Tp __d,
    601                                  memory_order __s, memory_order __f) volatile _NOEXCEPT
    602         {return __c11_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);}
    603     _LIBCPP_INLINE_VISIBILITY
    604     bool compare_exchange_strong(_Tp& __e, _Tp __d,
    605                                  memory_order __s, memory_order __f) _NOEXCEPT
    606         {return __c11_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);}
    607     _LIBCPP_INLINE_VISIBILITY
    608     bool compare_exchange_weak(_Tp& __e, _Tp __d,
    609                               memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
    610         {return __c11_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);}
    611     _LIBCPP_INLINE_VISIBILITY
    612     bool compare_exchange_weak(_Tp& __e, _Tp __d,
    613                                memory_order __m = memory_order_seq_cst) _NOEXCEPT
    614         {return __c11_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);}
    615     _LIBCPP_INLINE_VISIBILITY
    616     bool compare_exchange_strong(_Tp& __e, _Tp __d,
    617                               memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
    618         {return __c11_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);}
    619     _LIBCPP_INLINE_VISIBILITY
    620     bool compare_exchange_strong(_Tp& __e, _Tp __d,
    621                                  memory_order __m = memory_order_seq_cst) _NOEXCEPT
    622         {return __c11_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);}
    623 
    624     _LIBCPP_INLINE_VISIBILITY
    625 #ifndef _LIBCPP_HAS_NO_DEFAULTED_FUNCTIONS
    626     __atomic_base() _NOEXCEPT = default;
    627 #else
    628     __atomic_base() _NOEXCEPT : __a_() {}
    629 #endif // _LIBCPP_HAS_NO_DEFAULTED_FUNCTIONS
    630 
    631     _LIBCPP_INLINE_VISIBILITY
    632     _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {}
    633 #ifndef _LIBCPP_HAS_NO_DELETED_FUNCTIONS
    634     __atomic_base(const __atomic_base&) = delete;
    635     __atomic_base& operator=(const __atomic_base&) = delete;
    636     __atomic_base& operator=(const __atomic_base&) volatile = delete;
    637 #else  // _LIBCPP_HAS_NO_DELETED_FUNCTIONS
    638 private:
    639     __atomic_base(const __atomic_base&);
    640     __atomic_base& operator=(const __atomic_base&);
    641     __atomic_base& operator=(const __atomic_base&) volatile;
    642 #endif  // _LIBCPP_HAS_NO_DELETED_FUNCTIONS
    643 };
    644 
    645 // atomic<Integral>
    646 
    647 template <class _Tp>
    648 struct __atomic_base<_Tp, true>
    649     : public __atomic_base<_Tp, false>
    650 {
    651     typedef __atomic_base<_Tp, false> __base;
    652     _LIBCPP_INLINE_VISIBILITY
    653     __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT
    654     _LIBCPP_INLINE_VISIBILITY
    655     _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {}
    656 
    657     _LIBCPP_INLINE_VISIBILITY
    658     _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
    659         {return __c11_atomic_fetch_add(&this->__a_, __op, __m);}
    660     _LIBCPP_INLINE_VISIBILITY
    661     _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
    662         {return __c11_atomic_fetch_add(&this->__a_, __op, __m);}
    663     _LIBCPP_INLINE_VISIBILITY
    664     _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
    665         {return __c11_atomic_fetch_sub(&this->__a_, __op, __m);}
    666     _LIBCPP_INLINE_VISIBILITY
    667     _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
    668         {return __c11_atomic_fetch_sub(&this->__a_, __op, __m);}
    669     _LIBCPP_INLINE_VISIBILITY
    670     _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
    671         {return __c11_atomic_fetch_and(&this->__a_, __op, __m);}
    672     _LIBCPP_INLINE_VISIBILITY
    673     _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
    674         {return __c11_atomic_fetch_and(&this->__a_, __op, __m);}
    675     _LIBCPP_INLINE_VISIBILITY
    676     _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
    677         {return __c11_atomic_fetch_or(&this->__a_, __op, __m);}
    678     _LIBCPP_INLINE_VISIBILITY
    679     _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
    680         {return __c11_atomic_fetch_or(&this->__a_, __op, __m);}
    681     _LIBCPP_INLINE_VISIBILITY
    682     _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
    683         {return __c11_atomic_fetch_xor(&this->__a_, __op, __m);}
    684     _LIBCPP_INLINE_VISIBILITY
    685     _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
    686         {return __c11_atomic_fetch_xor(&this->__a_, __op, __m);}
    687 
    688     _LIBCPP_INLINE_VISIBILITY
    689     _Tp operator++(int) volatile _NOEXCEPT      {return fetch_add(_Tp(1));}
    690     _LIBCPP_INLINE_VISIBILITY
    691     _Tp operator++(int) _NOEXCEPT               {return fetch_add(_Tp(1));}
    692     _LIBCPP_INLINE_VISIBILITY
    693     _Tp operator--(int) volatile _NOEXCEPT      {return fetch_sub(_Tp(1));}
    694     _LIBCPP_INLINE_VISIBILITY
    695     _Tp operator--(int) _NOEXCEPT               {return fetch_sub(_Tp(1));}
    696     _LIBCPP_INLINE_VISIBILITY
    697     _Tp operator++() volatile _NOEXCEPT         {return fetch_add(_Tp(1)) + _Tp(1);}
    698     _LIBCPP_INLINE_VISIBILITY
    699     _Tp operator++() _NOEXCEPT                  {return fetch_add(_Tp(1)) + _Tp(1);}
    700     _LIBCPP_INLINE_VISIBILITY
    701     _Tp operator--() volatile _NOEXCEPT         {return fetch_sub(_Tp(1)) - _Tp(1);}
    702     _LIBCPP_INLINE_VISIBILITY
    703     _Tp operator--() _NOEXCEPT                  {return fetch_sub(_Tp(1)) - _Tp(1);}
    704     _LIBCPP_INLINE_VISIBILITY
    705     _Tp operator+=(_Tp __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;}
    706     _LIBCPP_INLINE_VISIBILITY
    707     _Tp operator+=(_Tp __op) _NOEXCEPT          {return fetch_add(__op) + __op;}
    708     _LIBCPP_INLINE_VISIBILITY
    709     _Tp operator-=(_Tp __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;}
    710     _LIBCPP_INLINE_VISIBILITY
    711     _Tp operator-=(_Tp __op) _NOEXCEPT          {return fetch_sub(__op) - __op;}
    712     _LIBCPP_INLINE_VISIBILITY
    713     _Tp operator&=(_Tp __op) volatile _NOEXCEPT {return fetch_and(__op) & __op;}
    714     _LIBCPP_INLINE_VISIBILITY
    715     _Tp operator&=(_Tp __op) _NOEXCEPT          {return fetch_and(__op) & __op;}
    716     _LIBCPP_INLINE_VISIBILITY
    717     _Tp operator|=(_Tp __op) volatile _NOEXCEPT {return fetch_or(__op) | __op;}
    718     _LIBCPP_INLINE_VISIBILITY
    719     _Tp operator|=(_Tp __op) _NOEXCEPT          {return fetch_or(__op) | __op;}
    720     _LIBCPP_INLINE_VISIBILITY
    721     _Tp operator^=(_Tp __op) volatile _NOEXCEPT {return fetch_xor(__op) ^ __op;}
    722     _LIBCPP_INLINE_VISIBILITY
    723     _Tp operator^=(_Tp __op) _NOEXCEPT          {return fetch_xor(__op) ^ __op;}
    724 };
    725 
    726 // atomic<T>
    727 
    728 template <class _Tp>
    729 struct atomic
    730     : public __atomic_base<_Tp>
    731 {
    732     typedef __atomic_base<_Tp> __base;
    733     _LIBCPP_INLINE_VISIBILITY
    734     atomic() _NOEXCEPT _LIBCPP_DEFAULT
    735     _LIBCPP_INLINE_VISIBILITY
    736     _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {}
    737 
    738     _LIBCPP_INLINE_VISIBILITY
    739     _Tp operator=(_Tp __d) volatile _NOEXCEPT
    740         {__base::store(__d); return __d;}
    741     _LIBCPP_INLINE_VISIBILITY
    742     _Tp operator=(_Tp __d) _NOEXCEPT
    743         {__base::store(__d); return __d;}
    744 };
    745 
    746 // atomic<T*>
    747 
    748 template <class _Tp>
    749 struct atomic<_Tp*>
    750     : public __atomic_base<_Tp*>
    751 {
    752     typedef __atomic_base<_Tp*> __base;
    753     _LIBCPP_INLINE_VISIBILITY
    754     atomic() _NOEXCEPT _LIBCPP_DEFAULT
    755     _LIBCPP_INLINE_VISIBILITY
    756     _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {}
    757 
    758     _LIBCPP_INLINE_VISIBILITY
    759     _Tp* operator=(_Tp* __d) volatile _NOEXCEPT
    760         {__base::store(__d); return __d;}
    761     _LIBCPP_INLINE_VISIBILITY
    762     _Tp* operator=(_Tp* __d) _NOEXCEPT
    763         {__base::store(__d); return __d;}
    764 
    765     _LIBCPP_INLINE_VISIBILITY
    766     _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst)
    767                                                                         volatile _NOEXCEPT
    768         {return __c11_atomic_fetch_add(&this->__a_, __op, __m);}
    769     _LIBCPP_INLINE_VISIBILITY
    770     _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
    771         {return __c11_atomic_fetch_add(&this->__a_, __op, __m);}
    772     _LIBCPP_INLINE_VISIBILITY
    773     _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst)
    774                                                                         volatile _NOEXCEPT
    775         {return __c11_atomic_fetch_sub(&this->__a_, __op, __m);}
    776     _LIBCPP_INLINE_VISIBILITY
    777     _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
    778         {return __c11_atomic_fetch_sub(&this->__a_, __op, __m);}
    779 
    780     _LIBCPP_INLINE_VISIBILITY
    781     _Tp* operator++(int) volatile _NOEXCEPT            {return fetch_add(1);}
    782     _LIBCPP_INLINE_VISIBILITY
    783     _Tp* operator++(int) _NOEXCEPT                     {return fetch_add(1);}
    784     _LIBCPP_INLINE_VISIBILITY
    785     _Tp* operator--(int) volatile _NOEXCEPT            {return fetch_sub(1);}
    786     _LIBCPP_INLINE_VISIBILITY
    787     _Tp* operator--(int) _NOEXCEPT                     {return fetch_sub(1);}
    788     _LIBCPP_INLINE_VISIBILITY
    789     _Tp* operator++() volatile _NOEXCEPT               {return fetch_add(1) + 1;}
    790     _LIBCPP_INLINE_VISIBILITY
    791     _Tp* operator++() _NOEXCEPT                        {return fetch_add(1) + 1;}
    792     _LIBCPP_INLINE_VISIBILITY
    793     _Tp* operator--() volatile _NOEXCEPT               {return fetch_sub(1) - 1;}
    794     _LIBCPP_INLINE_VISIBILITY
    795     _Tp* operator--() _NOEXCEPT                        {return fetch_sub(1) - 1;}
    796     _LIBCPP_INLINE_VISIBILITY
    797     _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;}
    798     _LIBCPP_INLINE_VISIBILITY
    799     _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT          {return fetch_add(__op) + __op;}
    800     _LIBCPP_INLINE_VISIBILITY
    801     _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;}
    802     _LIBCPP_INLINE_VISIBILITY
    803     _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT          {return fetch_sub(__op) - __op;}
    804 };
    805 
    806 // atomic_is_lock_free
    807 
    808 template <class _Tp>
    809 inline _LIBCPP_INLINE_VISIBILITY
    810 bool
    811 atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT
    812 {
    813     return __o->is_lock_free();
    814 }
    815 
    816 template <class _Tp>
    817 inline _LIBCPP_INLINE_VISIBILITY
    818 bool
    819 atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT
    820 {
    821     return __o->is_lock_free();
    822 }
    823 
    824 // atomic_init
    825 
    826 template <class _Tp>
    827 inline _LIBCPP_INLINE_VISIBILITY
    828 void
    829 atomic_init(volatile atomic<_Tp>* __o, _Tp __d) _NOEXCEPT
    830 {
    831     __c11_atomic_init(&__o->__a_, __d);
    832 }
    833 
    834 template <class _Tp>
    835 inline _LIBCPP_INLINE_VISIBILITY
    836 void
    837 atomic_init(atomic<_Tp>* __o, _Tp __d) _NOEXCEPT
    838 {
    839     __c11_atomic_init(&__o->__a_, __d);
    840 }
    841 
    842 // atomic_store
    843 
    844 template <class _Tp>
    845 inline _LIBCPP_INLINE_VISIBILITY
    846 void
    847 atomic_store(volatile atomic<_Tp>* __o, _Tp __d) _NOEXCEPT
    848 {
    849     __o->store(__d);
    850 }
    851 
    852 template <class _Tp>
    853 inline _LIBCPP_INLINE_VISIBILITY
    854 void
    855 atomic_store(atomic<_Tp>* __o, _Tp __d) _NOEXCEPT
    856 {
    857     __o->store(__d);
    858 }
    859 
    860 // atomic_store_explicit
    861 
    862 template <class _Tp>
    863 inline _LIBCPP_INLINE_VISIBILITY
    864 void
    865 atomic_store_explicit(volatile atomic<_Tp>* __o, _Tp __d, memory_order __m) _NOEXCEPT
    866 {
    867     __o->store(__d, __m);
    868 }
    869 
    870 template <class _Tp>
    871 inline _LIBCPP_INLINE_VISIBILITY
    872 void
    873 atomic_store_explicit(atomic<_Tp>* __o, _Tp __d, memory_order __m) _NOEXCEPT
    874 {
    875     __o->store(__d, __m);
    876 }
    877 
    878 // atomic_load
    879 
    880 template <class _Tp>
    881 inline _LIBCPP_INLINE_VISIBILITY
    882 _Tp
    883 atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT
    884 {
    885     return __o->load();
    886 }
    887 
    888 template <class _Tp>
    889 inline _LIBCPP_INLINE_VISIBILITY
    890 _Tp
    891 atomic_load(const atomic<_Tp>* __o) _NOEXCEPT
    892 {
    893     return __o->load();
    894 }
    895 
    896 // atomic_load_explicit
    897 
    898 template <class _Tp>
    899 inline _LIBCPP_INLINE_VISIBILITY
    900 _Tp
    901 atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT
    902 {
    903     return __o->load(__m);
    904 }
    905 
    906 template <class _Tp>
    907 inline _LIBCPP_INLINE_VISIBILITY
    908 _Tp
    909 atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT
    910 {
    911     return __o->load(__m);
    912 }
    913 
    914 // atomic_exchange
    915 
    916 template <class _Tp>
    917 inline _LIBCPP_INLINE_VISIBILITY
    918 _Tp
    919 atomic_exchange(volatile atomic<_Tp>* __o, _Tp __d) _NOEXCEPT
    920 {
    921     return __o->exchange(__d);
    922 }
    923 
    924 template <class _Tp>
    925 inline _LIBCPP_INLINE_VISIBILITY
    926 _Tp
    927 atomic_exchange(atomic<_Tp>* __o, _Tp __d) _NOEXCEPT
    928 {
    929     return __o->exchange(__d);
    930 }
    931 
    932 // atomic_exchange_explicit
    933 
    934 template <class _Tp>
    935 inline _LIBCPP_INLINE_VISIBILITY
    936 _Tp
    937 atomic_exchange_explicit(volatile atomic<_Tp>* __o, _Tp __d, memory_order __m) _NOEXCEPT
    938 {
    939     return __o->exchange(__d, __m);
    940 }
    941 
    942 template <class _Tp>
    943 inline _LIBCPP_INLINE_VISIBILITY
    944 _Tp
    945 atomic_exchange_explicit(atomic<_Tp>* __o, _Tp __d, memory_order __m) _NOEXCEPT
    946 {
    947     return __o->exchange(__d, __m);
    948 }
    949 
    950 // atomic_compare_exchange_weak
    951 
    952 template <class _Tp>
    953 inline _LIBCPP_INLINE_VISIBILITY
    954 bool
    955 atomic_compare_exchange_weak(volatile atomic<_Tp>* __o, _Tp* __e, _Tp __d) _NOEXCEPT
    956 {
    957     return __o->compare_exchange_weak(*__e, __d);
    958 }
    959 
    960 template <class _Tp>
    961 inline _LIBCPP_INLINE_VISIBILITY
    962 bool
    963 atomic_compare_exchange_weak(atomic<_Tp>* __o, _Tp* __e, _Tp __d) _NOEXCEPT
    964 {
    965     return __o->compare_exchange_weak(*__e, __d);
    966 }
    967 
    968 // atomic_compare_exchange_strong
    969 
    970 template <class _Tp>
    971 inline _LIBCPP_INLINE_VISIBILITY
    972 bool
    973 atomic_compare_exchange_strong(volatile atomic<_Tp>* __o, _Tp* __e, _Tp __d) _NOEXCEPT
    974 {
    975     return __o->compare_exchange_strong(*__e, __d);
    976 }
    977 
    978 template <class _Tp>
    979 inline _LIBCPP_INLINE_VISIBILITY
    980 bool
    981 atomic_compare_exchange_strong(atomic<_Tp>* __o, _Tp* __e, _Tp __d) _NOEXCEPT
    982 {
    983     return __o->compare_exchange_strong(*__e, __d);
    984 }
    985 
    986 // atomic_compare_exchange_weak_explicit
    987 
    988 template <class _Tp>
    989 inline _LIBCPP_INLINE_VISIBILITY
    990 bool
    991 atomic_compare_exchange_weak_explicit(volatile atomic<_Tp>* __o, _Tp* __e,
    992                                       _Tp __d,
    993                                       memory_order __s, memory_order __f) _NOEXCEPT
    994 {
    995     return __o->compare_exchange_weak(*__e, __d, __s, __f);
    996 }
    997 
    998 template <class _Tp>
    999 inline _LIBCPP_INLINE_VISIBILITY
   1000 bool
   1001 atomic_compare_exchange_weak_explicit(atomic<_Tp>* __o, _Tp* __e, _Tp __d,
   1002                                       memory_order __s, memory_order __f) _NOEXCEPT
   1003 {
   1004     return __o->compare_exchange_weak(*__e, __d, __s, __f);
   1005 }
   1006 
   1007 // atomic_compare_exchange_strong_explicit
   1008 
   1009 template <class _Tp>
   1010 inline _LIBCPP_INLINE_VISIBILITY
   1011 bool
   1012 atomic_compare_exchange_strong_explicit(volatile atomic<_Tp>* __o,
   1013                                         _Tp* __e, _Tp __d,
   1014                                         memory_order __s, memory_order __f) _NOEXCEPT
   1015 {
   1016     return __o->compare_exchange_strong(*__e, __d, __s, __f);
   1017 }
   1018 
   1019 template <class _Tp>
   1020 inline _LIBCPP_INLINE_VISIBILITY
   1021 bool
   1022 atomic_compare_exchange_strong_explicit(atomic<_Tp>* __o, _Tp* __e,
   1023                                         _Tp __d,
   1024                                         memory_order __s, memory_order __f) _NOEXCEPT
   1025 {
   1026     return __o->compare_exchange_strong(*__e, __d, __s, __f);
   1027 }
   1028 
   1029 // atomic_fetch_add
   1030 
   1031 template <class _Tp>
   1032 inline _LIBCPP_INLINE_VISIBILITY
   1033 typename enable_if
   1034 <
   1035     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1036     _Tp
   1037 >::type
   1038 atomic_fetch_add(volatile atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
   1039 {
   1040     return __o->fetch_add(__op);
   1041 }
   1042 
   1043 template <class _Tp>
   1044 inline _LIBCPP_INLINE_VISIBILITY
   1045 typename enable_if
   1046 <
   1047     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1048     _Tp
   1049 >::type
   1050 atomic_fetch_add(atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
   1051 {
   1052     return __o->fetch_add(__op);
   1053 }
   1054 
   1055 template <class _Tp>
   1056 inline _LIBCPP_INLINE_VISIBILITY
   1057 _Tp*
   1058 atomic_fetch_add(volatile atomic<_Tp*>* __o, ptrdiff_t __op) _NOEXCEPT
   1059 {
   1060     return __o->fetch_add(__op);
   1061 }
   1062 
   1063 template <class _Tp>
   1064 inline _LIBCPP_INLINE_VISIBILITY
   1065 _Tp*
   1066 atomic_fetch_add(atomic<_Tp*>* __o, ptrdiff_t __op) _NOEXCEPT
   1067 {
   1068     return __o->fetch_add(__op);
   1069 }
   1070 
   1071 // atomic_fetch_add_explicit
   1072 
   1073 template <class _Tp>
   1074 inline _LIBCPP_INLINE_VISIBILITY
   1075 typename enable_if
   1076 <
   1077     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1078     _Tp
   1079 >::type
   1080 atomic_fetch_add_explicit(volatile atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
   1081 {
   1082     return __o->fetch_add(__op, __m);
   1083 }
   1084 
   1085 template <class _Tp>
   1086 inline _LIBCPP_INLINE_VISIBILITY
   1087 typename enable_if
   1088 <
   1089     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1090     _Tp
   1091 >::type
   1092 atomic_fetch_add_explicit(atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
   1093 {
   1094     return __o->fetch_add(__op, __m);
   1095 }
   1096 
   1097 template <class _Tp>
   1098 inline _LIBCPP_INLINE_VISIBILITY
   1099 _Tp*
   1100 atomic_fetch_add_explicit(volatile atomic<_Tp*>* __o, ptrdiff_t __op,
   1101                           memory_order __m) _NOEXCEPT
   1102 {
   1103     return __o->fetch_add(__op, __m);
   1104 }
   1105 
   1106 template <class _Tp>
   1107 inline _LIBCPP_INLINE_VISIBILITY
   1108 _Tp*
   1109 atomic_fetch_add_explicit(atomic<_Tp*>* __o, ptrdiff_t __op, memory_order __m) _NOEXCEPT
   1110 {
   1111     return __o->fetch_add(__op, __m);
   1112 }
   1113 
   1114 // atomic_fetch_sub
   1115 
   1116 template <class _Tp>
   1117 inline _LIBCPP_INLINE_VISIBILITY
   1118 typename enable_if
   1119 <
   1120     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1121     _Tp
   1122 >::type
   1123 atomic_fetch_sub(volatile atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
   1124 {
   1125     return __o->fetch_sub(__op);
   1126 }
   1127 
   1128 template <class _Tp>
   1129 inline _LIBCPP_INLINE_VISIBILITY
   1130 typename enable_if
   1131 <
   1132     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1133     _Tp
   1134 >::type
   1135 atomic_fetch_sub(atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
   1136 {
   1137     return __o->fetch_sub(__op);
   1138 }
   1139 
   1140 template <class _Tp>
   1141 inline _LIBCPP_INLINE_VISIBILITY
   1142 _Tp*
   1143 atomic_fetch_sub(volatile atomic<_Tp*>* __o, ptrdiff_t __op) _NOEXCEPT
   1144 {
   1145     return __o->fetch_sub(__op);
   1146 }
   1147 
   1148 template <class _Tp>
   1149 inline _LIBCPP_INLINE_VISIBILITY
   1150 _Tp*
   1151 atomic_fetch_sub(atomic<_Tp*>* __o, ptrdiff_t __op) _NOEXCEPT
   1152 {
   1153     return __o->fetch_sub(__op);
   1154 }
   1155 
   1156 // atomic_fetch_sub_explicit
   1157 
   1158 template <class _Tp>
   1159 inline _LIBCPP_INLINE_VISIBILITY
   1160 typename enable_if
   1161 <
   1162     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1163     _Tp
   1164 >::type
   1165 atomic_fetch_sub_explicit(volatile atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
   1166 {
   1167     return __o->fetch_sub(__op, __m);
   1168 }
   1169 
   1170 template <class _Tp>
   1171 inline _LIBCPP_INLINE_VISIBILITY
   1172 typename enable_if
   1173 <
   1174     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1175     _Tp
   1176 >::type
   1177 atomic_fetch_sub_explicit(atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
   1178 {
   1179     return __o->fetch_sub(__op, __m);
   1180 }
   1181 
   1182 template <class _Tp>
   1183 inline _LIBCPP_INLINE_VISIBILITY
   1184 _Tp*
   1185 atomic_fetch_sub_explicit(volatile atomic<_Tp*>* __o, ptrdiff_t __op,
   1186                           memory_order __m) _NOEXCEPT
   1187 {
   1188     return __o->fetch_sub(__op, __m);
   1189 }
   1190 
   1191 template <class _Tp>
   1192 inline _LIBCPP_INLINE_VISIBILITY
   1193 _Tp*
   1194 atomic_fetch_sub_explicit(atomic<_Tp*>* __o, ptrdiff_t __op, memory_order __m) _NOEXCEPT
   1195 {
   1196     return __o->fetch_sub(__op, __m);
   1197 }
   1198 
   1199 // atomic_fetch_and
   1200 
   1201 template <class _Tp>
   1202 inline _LIBCPP_INLINE_VISIBILITY
   1203 typename enable_if
   1204 <
   1205     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1206     _Tp
   1207 >::type
   1208 atomic_fetch_and(volatile atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
   1209 {
   1210     return __o->fetch_and(__op);
   1211 }
   1212 
   1213 template <class _Tp>
   1214 inline _LIBCPP_INLINE_VISIBILITY
   1215 typename enable_if
   1216 <
   1217     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1218     _Tp
   1219 >::type
   1220 atomic_fetch_and(atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
   1221 {
   1222     return __o->fetch_and(__op);
   1223 }
   1224 
   1225 // atomic_fetch_and_explicit
   1226 
   1227 template <class _Tp>
   1228 inline _LIBCPP_INLINE_VISIBILITY
   1229 typename enable_if
   1230 <
   1231     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1232     _Tp
   1233 >::type
   1234 atomic_fetch_and_explicit(volatile atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
   1235 {
   1236     return __o->fetch_and(__op, __m);
   1237 }
   1238 
   1239 template <class _Tp>
   1240 inline _LIBCPP_INLINE_VISIBILITY
   1241 typename enable_if
   1242 <
   1243     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1244     _Tp
   1245 >::type
   1246 atomic_fetch_and_explicit(atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
   1247 {
   1248     return __o->fetch_and(__op, __m);
   1249 }
   1250 
   1251 // atomic_fetch_or
   1252 
   1253 template <class _Tp>
   1254 inline _LIBCPP_INLINE_VISIBILITY
   1255 typename enable_if
   1256 <
   1257     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1258     _Tp
   1259 >::type
   1260 atomic_fetch_or(volatile atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
   1261 {
   1262     return __o->fetch_or(__op);
   1263 }
   1264 
   1265 template <class _Tp>
   1266 inline _LIBCPP_INLINE_VISIBILITY
   1267 typename enable_if
   1268 <
   1269     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1270     _Tp
   1271 >::type
   1272 atomic_fetch_or(atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
   1273 {
   1274     return __o->fetch_or(__op);
   1275 }
   1276 
   1277 // atomic_fetch_or_explicit
   1278 
   1279 template <class _Tp>
   1280 inline _LIBCPP_INLINE_VISIBILITY
   1281 typename enable_if
   1282 <
   1283     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1284     _Tp
   1285 >::type
   1286 atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
   1287 {
   1288     return __o->fetch_or(__op, __m);
   1289 }
   1290 
   1291 template <class _Tp>
   1292 inline _LIBCPP_INLINE_VISIBILITY
   1293 typename enable_if
   1294 <
   1295     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1296     _Tp
   1297 >::type
   1298 atomic_fetch_or_explicit(atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
   1299 {
   1300     return __o->fetch_or(__op, __m);
   1301 }
   1302 
   1303 // atomic_fetch_xor
   1304 
   1305 template <class _Tp>
   1306 inline _LIBCPP_INLINE_VISIBILITY
   1307 typename enable_if
   1308 <
   1309     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1310     _Tp
   1311 >::type
   1312 atomic_fetch_xor(volatile atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
   1313 {
   1314     return __o->fetch_xor(__op);
   1315 }
   1316 
   1317 template <class _Tp>
   1318 inline _LIBCPP_INLINE_VISIBILITY
   1319 typename enable_if
   1320 <
   1321     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1322     _Tp
   1323 >::type
   1324 atomic_fetch_xor(atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
   1325 {
   1326     return __o->fetch_xor(__op);
   1327 }
   1328 
   1329 // atomic_fetch_xor_explicit
   1330 
   1331 template <class _Tp>
   1332 inline _LIBCPP_INLINE_VISIBILITY
   1333 typename enable_if
   1334 <
   1335     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1336     _Tp
   1337 >::type
   1338 atomic_fetch_xor_explicit(volatile atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
   1339 {
   1340     return __o->fetch_xor(__op, __m);
   1341 }
   1342 
   1343 template <class _Tp>
   1344 inline _LIBCPP_INLINE_VISIBILITY
   1345 typename enable_if
   1346 <
   1347     is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
   1348     _Tp
   1349 >::type
   1350 atomic_fetch_xor_explicit(atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
   1351 {
   1352     return __o->fetch_xor(__op, __m);
   1353 }
   1354 
   1355 // flag type and operations
   1356 
   1357 typedef struct atomic_flag
   1358 {
   1359     _Atomic(bool) __a_;
   1360 
   1361     _LIBCPP_INLINE_VISIBILITY
   1362     bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
   1363         {return __c11_atomic_exchange(&__a_, true, __m);}
   1364     _LIBCPP_INLINE_VISIBILITY
   1365     bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT
   1366         {return __c11_atomic_exchange(&__a_, true, __m);}
   1367     _LIBCPP_INLINE_VISIBILITY
   1368     void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
   1369         {__c11_atomic_store(&__a_, false, __m);}
   1370     _LIBCPP_INLINE_VISIBILITY
   1371     void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT
   1372         {__c11_atomic_store(&__a_, false, __m);}
   1373 
   1374     _LIBCPP_INLINE_VISIBILITY
   1375 #ifndef _LIBCPP_HAS_NO_DEFAULTED_FUNCTIONS
   1376     atomic_flag() _NOEXCEPT = default;
   1377 #else
   1378     atomic_flag() _NOEXCEPT : __a_() {}
   1379 #endif // _LIBCPP_HAS_NO_DEFAULTED_FUNCTIONS
   1380 
   1381     _LIBCPP_INLINE_VISIBILITY
   1382     atomic_flag(bool __b) _NOEXCEPT : __a_(__b) {}
   1383 
   1384 #ifndef _LIBCPP_HAS_NO_DELETED_FUNCTIONS
   1385     atomic_flag(const atomic_flag&) = delete;
   1386     atomic_flag& operator=(const atomic_flag&) = delete;
   1387     atomic_flag& operator=(const atomic_flag&) volatile = delete;
   1388 #else  // _LIBCPP_HAS_NO_DELETED_FUNCTIONS
   1389 private:
   1390     atomic_flag(const atomic_flag&);
   1391     atomic_flag& operator=(const atomic_flag&);
   1392     atomic_flag& operator=(const atomic_flag&) volatile;
   1393 #endif  // _LIBCPP_HAS_NO_DELETED_FUNCTIONS
   1394 } atomic_flag;
   1395 
   1396 inline _LIBCPP_INLINE_VISIBILITY
   1397 bool
   1398 atomic_flag_test_and_set(volatile atomic_flag* __o) _NOEXCEPT
   1399 {
   1400     return __o->test_and_set();
   1401 }
   1402 
   1403 inline _LIBCPP_INLINE_VISIBILITY
   1404 bool
   1405 atomic_flag_test_and_set(atomic_flag* __o) _NOEXCEPT
   1406 {
   1407     return __o->test_and_set();
   1408 }
   1409 
   1410 inline _LIBCPP_INLINE_VISIBILITY
   1411 bool
   1412 atomic_flag_test_and_set_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT
   1413 {
   1414     return __o->test_and_set(__m);
   1415 }
   1416 
   1417 inline _LIBCPP_INLINE_VISIBILITY
   1418 bool
   1419 atomic_flag_test_and_set_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT
   1420 {
   1421     return __o->test_and_set(__m);
   1422 }
   1423 
   1424 inline _LIBCPP_INLINE_VISIBILITY
   1425 void
   1426 atomic_flag_clear(volatile atomic_flag* __o) _NOEXCEPT
   1427 {
   1428     __o->clear();
   1429 }
   1430 
   1431 inline _LIBCPP_INLINE_VISIBILITY
   1432 void
   1433 atomic_flag_clear(atomic_flag* __o) _NOEXCEPT
   1434 {
   1435     __o->clear();
   1436 }
   1437 
   1438 inline _LIBCPP_INLINE_VISIBILITY
   1439 void
   1440 atomic_flag_clear_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT
   1441 {
   1442     __o->clear(__m);
   1443 }
   1444 
   1445 inline _LIBCPP_INLINE_VISIBILITY
   1446 void
   1447 atomic_flag_clear_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT
   1448 {
   1449     __o->clear(__m);
   1450 }
   1451 
   1452 // fences
   1453 
   1454 inline _LIBCPP_INLINE_VISIBILITY
   1455 void
   1456 atomic_thread_fence(memory_order __m) _NOEXCEPT
   1457 {
   1458     __c11_atomic_thread_fence(__m);
   1459 }
   1460 
   1461 inline _LIBCPP_INLINE_VISIBILITY
   1462 void
   1463 atomic_signal_fence(memory_order __m) _NOEXCEPT
   1464 {
   1465     __c11_atomic_signal_fence(__m);
   1466 }
   1467 
   1468 // Atomics for standard typedef types
   1469 
   1470 typedef atomic<bool>               atomic_bool;
   1471 typedef atomic<char>               atomic_char;
   1472 typedef atomic<signed char>        atomic_schar;
   1473 typedef atomic<unsigned char>      atomic_uchar;
   1474 typedef atomic<short>              atomic_short;
   1475 typedef atomic<unsigned short>     atomic_ushort;
   1476 typedef atomic<int>                atomic_int;
   1477 typedef atomic<unsigned int>       atomic_uint;
   1478 typedef atomic<long>               atomic_long;
   1479 typedef atomic<unsigned long>      atomic_ulong;
   1480 typedef atomic<long long>          atomic_llong;
   1481 typedef atomic<unsigned long long> atomic_ullong;
   1482 typedef atomic<char16_t>           atomic_char16_t;
   1483 typedef atomic<char32_t>           atomic_char32_t;
   1484 typedef atomic<wchar_t>            atomic_wchar_t;
   1485 
   1486 typedef atomic<int_least8_t>   atomic_int_least8_t;
   1487 typedef atomic<uint_least8_t>  atomic_uint_least8_t;
   1488 typedef atomic<int_least16_t>  atomic_int_least16_t;
   1489 typedef atomic<uint_least16_t> atomic_uint_least16_t;
   1490 typedef atomic<int_least32_t>  atomic_int_least32_t;
   1491 typedef atomic<uint_least32_t> atomic_uint_least32_t;
   1492 typedef atomic<int_least64_t>  atomic_int_least64_t;
   1493 typedef atomic<uint_least64_t> atomic_uint_least64_t;
   1494 
   1495 typedef atomic<int_fast8_t>   atomic_int_fast8_t;
   1496 typedef atomic<uint_fast8_t>  atomic_uint_fast8_t;
   1497 typedef atomic<int_fast16_t>  atomic_int_fast16_t;
   1498 typedef atomic<uint_fast16_t> atomic_uint_fast16_t;
   1499 typedef atomic<int_fast32_t>  atomic_int_fast32_t;
   1500 typedef atomic<uint_fast32_t> atomic_uint_fast32_t;
   1501 typedef atomic<int_fast64_t>  atomic_int_fast64_t;
   1502 typedef atomic<uint_fast64_t> atomic_uint_fast64_t;
   1503 
   1504 typedef atomic<intptr_t>  atomic_intptr_t;
   1505 typedef atomic<uintptr_t> atomic_uintptr_t;
   1506 typedef atomic<size_t>    atomic_size_t;
   1507 typedef atomic<ptrdiff_t> atomic_ptrdiff_t;
   1508 typedef atomic<intmax_t>  atomic_intmax_t;
   1509 typedef atomic<uintmax_t> atomic_uintmax_t;
   1510 
   1511 #define ATOMIC_FLAG_INIT {false}
   1512 #define ATOMIC_VAR_INIT(__v) {__v}
   1513 
   1514 // lock-free property
   1515 
   1516 #define ATOMIC_BOOL_LOCK_FREE      __GCC_ATOMIC_BOOL_LOCK_FREE
   1517 #define ATOMIC_CHAR_LOCK_FREE      __GCC_ATOMIC_CHAR_LOCK_FREE
   1518 #define ATOMIC_CHAR16_T_LOCK_FREE  __GCC_ATOMIC_CHAR16_T_LOCK_FREE
   1519 #define ATOMIC_CHAR32_T_LOCK_FREE  __GCC_ATOMIC_CHAR32_T_LOCK_FREE
   1520 #define ATOMIC_WCHAR_T_LOCK_FREE   __GCC_ATOMIC_WCHAR_T_LOCK_FREE
   1521 #define ATOMIC_SHORT_LOCK_FREE     __GCC_ATOMIC_SHORT_LOCK_FREE
   1522 #define ATOMIC_INT_LOCK_FREE       __GCC_ATOMIC_INT_LOCK_FREE
   1523 #define ATOMIC_LONG_LOCK_FREE      __GCC_ATOMIC_LONG_LOCK_FREE
   1524 #define ATOMIC_LLONG_LOCK_FREE     __GCC_ATOMIC_LLONG_LOCK_FREE
   1525 #define ATOMIC_POINTER_LOCK_FREE   __GCC_ATOMIC_POINTER_LOCK_FREE
   1526 
   1527 #endif  //  !__has_feature(cxx_atomic)
   1528 
   1529 _LIBCPP_END_NAMESPACE_STD
   1530 
   1531 #endif  // _LIBCPP_ATOMIC
   1532