1 //===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file is a part of ThreadSanitizer/AddressSanitizer runtime. 11 // Not intended for direct inclusion. Include sanitizer_atomic.h. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #ifndef SANITIZER_ATOMIC_MSVC_H 16 #define SANITIZER_ATOMIC_MSVC_H 17 18 extern "C" void _ReadWriteBarrier(); 19 #pragma intrinsic(_ReadWriteBarrier) 20 extern "C" void _mm_mfence(); 21 #pragma intrinsic(_mm_mfence) 22 extern "C" void _mm_pause(); 23 #pragma intrinsic(_mm_pause) 24 extern "C" long _InterlockedExchangeAdd( // NOLINT 25 long volatile * Addend, long Value); // NOLINT 26 #pragma intrinsic(_InterlockedExchangeAdd) 27 extern "C" short _InterlockedCompareExchange16( // NOLINT 28 short volatile *Destination, // NOLINT 29 short Exchange, short Comparand); // NOLINT 30 #pragma intrinsic(_InterlockedCompareExchange16) 31 extern "C" 32 long long _InterlockedCompareExchange64( // NOLINT 33 long long volatile *Destination, // NOLINT 34 long long Exchange, long long Comparand); // NOLINT 35 #pragma intrinsic(_InterlockedCompareExchange64) 36 extern "C" void *_InterlockedCompareExchangePointer( 37 void *volatile *Destination, 38 void *Exchange, void *Comparand); 39 #pragma intrinsic(_InterlockedCompareExchangePointer) 40 extern "C" 41 long __cdecl _InterlockedCompareExchange( // NOLINT 42 long volatile *Destination, // NOLINT 43 long Exchange, long Comparand); // NOLINT 44 #pragma intrinsic(_InterlockedCompareExchange) 45 46 #ifdef _WIN64 47 extern "C" long long _InterlockedExchangeAdd64( // NOLINT 48 long long volatile * Addend, long long Value); // NOLINT 49 #pragma intrinsic(_InterlockedExchangeAdd64) 50 #endif 51 52 namespace __sanitizer { 53 54 INLINE void atomic_signal_fence(memory_order) { 55 _ReadWriteBarrier(); 56 } 57 58 INLINE void atomic_thread_fence(memory_order) { 59 _mm_mfence(); 60 } 61 62 INLINE void proc_yield(int cnt) { 63 for (int i = 0; i < cnt; i++) 64 _mm_pause(); 65 } 66 67 template<typename T> 68 INLINE typename T::Type atomic_load( 69 const volatile T *a, memory_order mo) { 70 DCHECK(mo & (memory_order_relaxed | memory_order_consume 71 | memory_order_acquire | memory_order_seq_cst)); 72 DCHECK(!((uptr)a % sizeof(*a))); 73 typename T::Type v; 74 // FIXME(dvyukov): 64-bit load is not atomic on 32-bits. 75 if (mo == memory_order_relaxed) { 76 v = a->val_dont_use; 77 } else { 78 atomic_signal_fence(memory_order_seq_cst); 79 v = a->val_dont_use; 80 atomic_signal_fence(memory_order_seq_cst); 81 } 82 return v; 83 } 84 85 template<typename T> 86 INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { 87 DCHECK(mo & (memory_order_relaxed | memory_order_release 88 | memory_order_seq_cst)); 89 DCHECK(!((uptr)a % sizeof(*a))); 90 // FIXME(dvyukov): 64-bit store is not atomic on 32-bits. 91 if (mo == memory_order_relaxed) { 92 a->val_dont_use = v; 93 } else { 94 atomic_signal_fence(memory_order_seq_cst); 95 a->val_dont_use = v; 96 atomic_signal_fence(memory_order_seq_cst); 97 } 98 if (mo == memory_order_seq_cst) 99 atomic_thread_fence(memory_order_seq_cst); 100 } 101 102 INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a, 103 u32 v, memory_order mo) { 104 (void)mo; 105 DCHECK(!((uptr)a % sizeof(*a))); 106 return (u32)_InterlockedExchangeAdd( 107 (volatile long*)&a->val_dont_use, (long)v); // NOLINT 108 } 109 110 INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a, 111 uptr v, memory_order mo) { 112 (void)mo; 113 DCHECK(!((uptr)a % sizeof(*a))); 114 #ifdef _WIN64 115 return (uptr)_InterlockedExchangeAdd64( 116 (volatile long long*)&a->val_dont_use, (long long)v); // NOLINT 117 #else 118 return (uptr)_InterlockedExchangeAdd( 119 (volatile long*)&a->val_dont_use, (long)v); // NOLINT 120 #endif 121 } 122 123 INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a, 124 u32 v, memory_order mo) { 125 (void)mo; 126 DCHECK(!((uptr)a % sizeof(*a))); 127 return (u32)_InterlockedExchangeAdd( 128 (volatile long*)&a->val_dont_use, -(long)v); // NOLINT 129 } 130 131 INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a, 132 uptr v, memory_order mo) { 133 (void)mo; 134 DCHECK(!((uptr)a % sizeof(*a))); 135 #ifdef _WIN64 136 return (uptr)_InterlockedExchangeAdd64( 137 (volatile long long*)&a->val_dont_use, -(long long)v); // NOLINT 138 #else 139 return (uptr)_InterlockedExchangeAdd( 140 (volatile long*)&a->val_dont_use, -(long)v); // NOLINT 141 #endif 142 } 143 144 INLINE u8 atomic_exchange(volatile atomic_uint8_t *a, 145 u8 v, memory_order mo) { 146 (void)mo; 147 DCHECK(!((uptr)a % sizeof(*a))); 148 __asm { 149 mov eax, a 150 mov cl, v 151 xchg [eax], cl // NOLINT 152 mov v, cl 153 } 154 return v; 155 } 156 157 INLINE u16 atomic_exchange(volatile atomic_uint16_t *a, 158 u16 v, memory_order mo) { 159 (void)mo; 160 DCHECK(!((uptr)a % sizeof(*a))); 161 __asm { 162 mov eax, a 163 mov cx, v 164 xchg [eax], cx // NOLINT 165 mov v, cx 166 } 167 return v; 168 } 169 170 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a, 171 u8 *cmp, 172 u8 xchgv, 173 memory_order mo) { 174 (void)mo; 175 DCHECK(!((uptr)a % sizeof(*a))); 176 u8 cmpv = *cmp; 177 u8 prev; 178 __asm { 179 mov al, cmpv 180 mov ecx, a 181 mov dl, xchgv 182 lock cmpxchg [ecx], dl 183 mov prev, al 184 } 185 if (prev == cmpv) 186 return true; 187 *cmp = prev; 188 return false; 189 } 190 191 INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a, 192 uptr *cmp, 193 uptr xchg, 194 memory_order mo) { 195 uptr cmpv = *cmp; 196 uptr prev = (uptr)_InterlockedCompareExchangePointer( 197 (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv); 198 if (prev == cmpv) 199 return true; 200 *cmp = prev; 201 return false; 202 } 203 204 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a, 205 u16 *cmp, 206 u16 xchg, 207 memory_order mo) { 208 u16 cmpv = *cmp; 209 u16 prev = (u16)_InterlockedCompareExchange16( 210 (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv); 211 if (prev == cmpv) 212 return true; 213 *cmp = prev; 214 return false; 215 } 216 217 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a, 218 u32 *cmp, 219 u32 xchg, 220 memory_order mo) { 221 u32 cmpv = *cmp; 222 u32 prev = (u32)_InterlockedCompareExchange( 223 (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv); 224 if (prev == cmpv) 225 return true; 226 *cmp = prev; 227 return false; 228 } 229 230 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a, 231 u64 *cmp, 232 u64 xchg, 233 memory_order mo) { 234 u64 cmpv = *cmp; 235 u64 prev = (u64)_InterlockedCompareExchange64( 236 (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv); 237 if (prev == cmpv) 238 return true; 239 *cmp = prev; 240 return false; 241 } 242 243 template<typename T> 244 INLINE bool atomic_compare_exchange_weak(volatile T *a, 245 typename T::Type *cmp, 246 typename T::Type xchg, 247 memory_order mo) { 248 return atomic_compare_exchange_strong(a, cmp, xchg, mo); 249 } 250 251 } // namespace __sanitizer 252 253 #endif // SANITIZER_ATOMIC_CLANG_H 254
