1 //===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file is a part of ThreadSanitizer/AddressSanitizer runtime. 11 // Not intended for direct inclusion. Include sanitizer_atomic.h. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #ifndef SANITIZER_ATOMIC_MSVC_H 16 #define SANITIZER_ATOMIC_MSVC_H 17 18 extern "C" void _ReadWriteBarrier(); 19 #pragma intrinsic(_ReadWriteBarrier) 20 extern "C" void _mm_mfence(); 21 #pragma intrinsic(_mm_mfence) 22 extern "C" void _mm_pause(); 23 #pragma intrinsic(_mm_pause) 24 extern "C" char _InterlockedExchange8( // NOLINT 25 char volatile *Addend, char Value); // NOLINT 26 #pragma intrinsic(_InterlockedExchange8) 27 extern "C" short _InterlockedExchange16( // NOLINT 28 short volatile *Addend, short Value); // NOLINT 29 #pragma intrinsic(_InterlockedExchange16) 30 extern "C" long _InterlockedExchange( // NOLINT 31 long volatile *Addend, long Value); // NOLINT 32 #pragma intrinsic(_InterlockedExchange) 33 extern "C" long _InterlockedExchangeAdd( // NOLINT 34 long volatile * Addend, long Value); // NOLINT 35 #pragma intrinsic(_InterlockedExchangeAdd) 36 extern "C" short _InterlockedCompareExchange16( // NOLINT 37 short volatile *Destination, // NOLINT 38 short Exchange, short Comparand); // NOLINT 39 #pragma intrinsic(_InterlockedCompareExchange16) 40 extern "C" 41 long long _InterlockedCompareExchange64( // NOLINT 42 long long volatile *Destination, // NOLINT 43 long long Exchange, long long Comparand); // NOLINT 44 #pragma intrinsic(_InterlockedCompareExchange64) 45 extern "C" void *_InterlockedCompareExchangePointer( 46 void *volatile *Destination, 47 void *Exchange, void *Comparand); 48 #pragma intrinsic(_InterlockedCompareExchangePointer) 49 extern "C" 50 long __cdecl _InterlockedCompareExchange( // NOLINT 51 long volatile *Destination, // NOLINT 52 long Exchange, long Comparand); // NOLINT 53 #pragma intrinsic(_InterlockedCompareExchange) 54 55 #ifdef _WIN64 56 extern "C" long long _InterlockedExchangeAdd64( // NOLINT 57 long long volatile * Addend, long long Value); // NOLINT 58 #pragma intrinsic(_InterlockedExchangeAdd64) 59 #endif 60 61 namespace __sanitizer { 62 63 INLINE void atomic_signal_fence(memory_order) { 64 _ReadWriteBarrier(); 65 } 66 67 INLINE void atomic_thread_fence(memory_order) { 68 _mm_mfence(); 69 } 70 71 INLINE void proc_yield(int cnt) { 72 for (int i = 0; i < cnt; i++) 73 _mm_pause(); 74 } 75 76 template<typename T> 77 INLINE typename T::Type atomic_load( 78 const volatile T *a, memory_order mo) { 79 DCHECK(mo & (memory_order_relaxed | memory_order_consume 80 | memory_order_acquire | memory_order_seq_cst)); 81 DCHECK(!((uptr)a % sizeof(*a))); 82 typename T::Type v; 83 // FIXME(dvyukov): 64-bit load is not atomic on 32-bits. 84 if (mo == memory_order_relaxed) { 85 v = a->val_dont_use; 86 } else { 87 atomic_signal_fence(memory_order_seq_cst); 88 v = a->val_dont_use; 89 atomic_signal_fence(memory_order_seq_cst); 90 } 91 return v; 92 } 93 94 template<typename T> 95 INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { 96 DCHECK(mo & (memory_order_relaxed | memory_order_release 97 | memory_order_seq_cst)); 98 DCHECK(!((uptr)a % sizeof(*a))); 99 // FIXME(dvyukov): 64-bit store is not atomic on 32-bits. 100 if (mo == memory_order_relaxed) { 101 a->val_dont_use = v; 102 } else { 103 atomic_signal_fence(memory_order_seq_cst); 104 a->val_dont_use = v; 105 atomic_signal_fence(memory_order_seq_cst); 106 } 107 if (mo == memory_order_seq_cst) 108 atomic_thread_fence(memory_order_seq_cst); 109 } 110 111 INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a, 112 u32 v, memory_order mo) { 113 (void)mo; 114 DCHECK(!((uptr)a % sizeof(*a))); 115 return (u32)_InterlockedExchangeAdd( 116 (volatile long*)&a->val_dont_use, (long)v); // NOLINT 117 } 118 119 INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a, 120 uptr v, memory_order mo) { 121 (void)mo; 122 DCHECK(!((uptr)a % sizeof(*a))); 123 #ifdef _WIN64 124 return (uptr)_InterlockedExchangeAdd64( 125 (volatile long long*)&a->val_dont_use, (long long)v); // NOLINT 126 #else 127 return (uptr)_InterlockedExchangeAdd( 128 (volatile long*)&a->val_dont_use, (long)v); // NOLINT 129 #endif 130 } 131 132 INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a, 133 u32 v, memory_order mo) { 134 (void)mo; 135 DCHECK(!((uptr)a % sizeof(*a))); 136 return (u32)_InterlockedExchangeAdd( 137 (volatile long*)&a->val_dont_use, -(long)v); // NOLINT 138 } 139 140 INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a, 141 uptr v, memory_order mo) { 142 (void)mo; 143 DCHECK(!((uptr)a % sizeof(*a))); 144 #ifdef _WIN64 145 return (uptr)_InterlockedExchangeAdd64( 146 (volatile long long*)&a->val_dont_use, -(long long)v); // NOLINT 147 #else 148 return (uptr)_InterlockedExchangeAdd( 149 (volatile long*)&a->val_dont_use, -(long)v); // NOLINT 150 #endif 151 } 152 153 INLINE u8 atomic_exchange(volatile atomic_uint8_t *a, 154 u8 v, memory_order mo) { 155 (void)mo; 156 DCHECK(!((uptr)a % sizeof(*a))); 157 return (u8)_InterlockedExchange8((volatile char*)&a->val_dont_use, v); 158 } 159 160 INLINE u16 atomic_exchange(volatile atomic_uint16_t *a, 161 u16 v, memory_order mo) { 162 (void)mo; 163 DCHECK(!((uptr)a % sizeof(*a))); 164 return (u16)_InterlockedExchange16((volatile short*)&a->val_dont_use, v); 165 } 166 167 INLINE u32 atomic_exchange(volatile atomic_uint32_t *a, 168 u32 v, memory_order mo) { 169 (void)mo; 170 DCHECK(!((uptr)a % sizeof(*a))); 171 return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v); 172 } 173 174 #ifndef _WIN64 175 176 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a, 177 u8 *cmp, 178 u8 xchgv, 179 memory_order mo) { 180 (void)mo; 181 DCHECK(!((uptr)a % sizeof(*a))); 182 u8 cmpv = *cmp; 183 u8 prev; 184 __asm { 185 mov al, cmpv 186 mov ecx, a 187 mov dl, xchgv 188 lock cmpxchg [ecx], dl 189 mov prev, al 190 } 191 if (prev == cmpv) 192 return true; 193 *cmp = prev; 194 return false; 195 } 196 197 #endif 198 199 INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a, 200 uptr *cmp, 201 uptr xchg, 202 memory_order mo) { 203 uptr cmpv = *cmp; 204 uptr prev = (uptr)_InterlockedCompareExchangePointer( 205 (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv); 206 if (prev == cmpv) 207 return true; 208 *cmp = prev; 209 return false; 210 } 211 212 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a, 213 u16 *cmp, 214 u16 xchg, 215 memory_order mo) { 216 u16 cmpv = *cmp; 217 u16 prev = (u16)_InterlockedCompareExchange16( 218 (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv); 219 if (prev == cmpv) 220 return true; 221 *cmp = prev; 222 return false; 223 } 224 225 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a, 226 u32 *cmp, 227 u32 xchg, 228 memory_order mo) { 229 u32 cmpv = *cmp; 230 u32 prev = (u32)_InterlockedCompareExchange( 231 (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv); 232 if (prev == cmpv) 233 return true; 234 *cmp = prev; 235 return false; 236 } 237 238 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a, 239 u64 *cmp, 240 u64 xchg, 241 memory_order mo) { 242 u64 cmpv = *cmp; 243 u64 prev = (u64)_InterlockedCompareExchange64( 244 (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv); 245 if (prev == cmpv) 246 return true; 247 *cmp = prev; 248 return false; 249 } 250 251 template<typename T> 252 INLINE bool atomic_compare_exchange_weak(volatile T *a, 253 typename T::Type *cmp, 254 typename T::Type xchg, 255 memory_order mo) { 256 return atomic_compare_exchange_strong(a, cmp, xchg, mo); 257 } 258 259 } // namespace __sanitizer 260 261 #endif // SANITIZER_ATOMIC_CLANG_H 262
