1 //===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file is a part of ThreadSanitizer/AddressSanitizer runtime. 11 // Not intended for direct inclusion. Include sanitizer_atomic.h. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #ifndef SANITIZER_ATOMIC_MSVC_H 16 #define SANITIZER_ATOMIC_MSVC_H 17 18 extern "C" void _ReadWriteBarrier(); 19 #pragma intrinsic(_ReadWriteBarrier) 20 extern "C" void _mm_mfence(); 21 #pragma intrinsic(_mm_mfence) 22 extern "C" void _mm_pause(); 23 #pragma intrinsic(_mm_pause) 24 extern "C" char _InterlockedExchange8( // NOLINT 25 char volatile *Addend, char Value); // NOLINT 26 #pragma intrinsic(_InterlockedExchange8) 27 extern "C" short _InterlockedExchange16( // NOLINT 28 short volatile *Addend, short Value); // NOLINT 29 #pragma intrinsic(_InterlockedExchange16) 30 extern "C" long _InterlockedExchange( // NOLINT 31 long volatile *Addend, long Value); // NOLINT 32 #pragma intrinsic(_InterlockedExchange) 33 extern "C" long _InterlockedExchangeAdd( // NOLINT 34 long volatile * Addend, long Value); // NOLINT 35 #pragma intrinsic(_InterlockedExchangeAdd) 36 extern "C" char _InterlockedCompareExchange8( // NOLINT 37 char volatile *Destination, // NOLINT 38 char Exchange, char Comparand); // NOLINT 39 #pragma intrinsic(_InterlockedCompareExchange8) 40 extern "C" short _InterlockedCompareExchange16( // NOLINT 41 short volatile *Destination, // NOLINT 42 short Exchange, short Comparand); // NOLINT 43 #pragma intrinsic(_InterlockedCompareExchange16) 44 extern "C" 45 long long _InterlockedCompareExchange64( // NOLINT 46 long long volatile *Destination, // NOLINT 47 long long Exchange, long long Comparand); // NOLINT 48 #pragma intrinsic(_InterlockedCompareExchange64) 49 extern "C" void *_InterlockedCompareExchangePointer( 50 void *volatile *Destination, 51 void *Exchange, void *Comparand); 52 #pragma intrinsic(_InterlockedCompareExchangePointer) 53 extern "C" 54 long __cdecl _InterlockedCompareExchange( // NOLINT 55 long volatile *Destination, // NOLINT 56 long Exchange, long Comparand); // NOLINT 57 #pragma intrinsic(_InterlockedCompareExchange) 58 59 #ifdef _WIN64 60 extern "C" long long _InterlockedExchangeAdd64( // NOLINT 61 long long volatile * Addend, long long Value); // NOLINT 62 #pragma intrinsic(_InterlockedExchangeAdd64) 63 #endif 64 65 namespace __sanitizer { 66 67 INLINE void atomic_signal_fence(memory_order) { 68 _ReadWriteBarrier(); 69 } 70 71 INLINE void atomic_thread_fence(memory_order) { 72 _mm_mfence(); 73 } 74 75 INLINE void proc_yield(int cnt) { 76 for (int i = 0; i < cnt; i++) 77 _mm_pause(); 78 } 79 80 template<typename T> 81 INLINE typename T::Type atomic_load( 82 const volatile T *a, memory_order mo) { 83 DCHECK(mo & (memory_order_relaxed | memory_order_consume 84 | memory_order_acquire | memory_order_seq_cst)); 85 DCHECK(!((uptr)a % sizeof(*a))); 86 typename T::Type v; 87 // FIXME(dvyukov): 64-bit load is not atomic on 32-bits. 88 if (mo == memory_order_relaxed) { 89 v = a->val_dont_use; 90 } else { 91 atomic_signal_fence(memory_order_seq_cst); 92 v = a->val_dont_use; 93 atomic_signal_fence(memory_order_seq_cst); 94 } 95 return v; 96 } 97 98 template<typename T> 99 INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { 100 DCHECK(mo & (memory_order_relaxed | memory_order_release 101 | memory_order_seq_cst)); 102 DCHECK(!((uptr)a % sizeof(*a))); 103 // FIXME(dvyukov): 64-bit store is not atomic on 32-bits. 104 if (mo == memory_order_relaxed) { 105 a->val_dont_use = v; 106 } else { 107 atomic_signal_fence(memory_order_seq_cst); 108 a->val_dont_use = v; 109 atomic_signal_fence(memory_order_seq_cst); 110 } 111 if (mo == memory_order_seq_cst) 112 atomic_thread_fence(memory_order_seq_cst); 113 } 114 115 INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a, 116 u32 v, memory_order mo) { 117 (void)mo; 118 DCHECK(!((uptr)a % sizeof(*a))); 119 return (u32)_InterlockedExchangeAdd( 120 (volatile long*)&a->val_dont_use, (long)v); // NOLINT 121 } 122 123 INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a, 124 uptr v, memory_order mo) { 125 (void)mo; 126 DCHECK(!((uptr)a % sizeof(*a))); 127 #ifdef _WIN64 128 return (uptr)_InterlockedExchangeAdd64( 129 (volatile long long*)&a->val_dont_use, (long long)v); // NOLINT 130 #else 131 return (uptr)_InterlockedExchangeAdd( 132 (volatile long*)&a->val_dont_use, (long)v); // NOLINT 133 #endif 134 } 135 136 INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a, 137 u32 v, memory_order mo) { 138 (void)mo; 139 DCHECK(!((uptr)a % sizeof(*a))); 140 return (u32)_InterlockedExchangeAdd( 141 (volatile long*)&a->val_dont_use, -(long)v); // NOLINT 142 } 143 144 INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a, 145 uptr v, memory_order mo) { 146 (void)mo; 147 DCHECK(!((uptr)a % sizeof(*a))); 148 #ifdef _WIN64 149 return (uptr)_InterlockedExchangeAdd64( 150 (volatile long long*)&a->val_dont_use, -(long long)v); // NOLINT 151 #else 152 return (uptr)_InterlockedExchangeAdd( 153 (volatile long*)&a->val_dont_use, -(long)v); // NOLINT 154 #endif 155 } 156 157 INLINE u8 atomic_exchange(volatile atomic_uint8_t *a, 158 u8 v, memory_order mo) { 159 (void)mo; 160 DCHECK(!((uptr)a % sizeof(*a))); 161 return (u8)_InterlockedExchange8((volatile char*)&a->val_dont_use, v); 162 } 163 164 INLINE u16 atomic_exchange(volatile atomic_uint16_t *a, 165 u16 v, memory_order mo) { 166 (void)mo; 167 DCHECK(!((uptr)a % sizeof(*a))); 168 return (u16)_InterlockedExchange16((volatile short*)&a->val_dont_use, v); 169 } 170 171 INLINE u32 atomic_exchange(volatile atomic_uint32_t *a, 172 u32 v, memory_order mo) { 173 (void)mo; 174 DCHECK(!((uptr)a % sizeof(*a))); 175 return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v); 176 } 177 178 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a, 179 u8 *cmp, 180 u8 xchgv, 181 memory_order mo) { 182 (void)mo; 183 DCHECK(!((uptr)a % sizeof(*a))); 184 u8 cmpv = *cmp; 185 #ifdef _WIN64 186 u8 prev = (u8)_InterlockedCompareExchange8( 187 (volatile char*)&a->val_dont_use, (char)xchgv, (char)cmpv); 188 #else 189 u8 prev; 190 __asm { 191 mov al, cmpv 192 mov ecx, a 193 mov dl, xchgv 194 lock cmpxchg [ecx], dl 195 mov prev, al 196 } 197 #endif 198 if (prev == cmpv) 199 return true; 200 *cmp = prev; 201 return false; 202 } 203 204 INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a, 205 uptr *cmp, 206 uptr xchg, 207 memory_order mo) { 208 uptr cmpv = *cmp; 209 uptr prev = (uptr)_InterlockedCompareExchangePointer( 210 (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv); 211 if (prev == cmpv) 212 return true; 213 *cmp = prev; 214 return false; 215 } 216 217 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a, 218 u16 *cmp, 219 u16 xchg, 220 memory_order mo) { 221 u16 cmpv = *cmp; 222 u16 prev = (u16)_InterlockedCompareExchange16( 223 (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv); 224 if (prev == cmpv) 225 return true; 226 *cmp = prev; 227 return false; 228 } 229 230 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a, 231 u32 *cmp, 232 u32 xchg, 233 memory_order mo) { 234 u32 cmpv = *cmp; 235 u32 prev = (u32)_InterlockedCompareExchange( 236 (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv); 237 if (prev == cmpv) 238 return true; 239 *cmp = prev; 240 return false; 241 } 242 243 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a, 244 u64 *cmp, 245 u64 xchg, 246 memory_order mo) { 247 u64 cmpv = *cmp; 248 u64 prev = (u64)_InterlockedCompareExchange64( 249 (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv); 250 if (prev == cmpv) 251 return true; 252 *cmp = prev; 253 return false; 254 } 255 256 template<typename T> 257 INLINE bool atomic_compare_exchange_weak(volatile T *a, 258 typename T::Type *cmp, 259 typename T::Type xchg, 260 memory_order mo) { 261 return atomic_compare_exchange_strong(a, cmp, xchg, mo); 262 } 263 264 } // namespace __sanitizer 265 266 #endif // SANITIZER_ATOMIC_CLANG_H 267
