1 /* 2 * Copyright (C) 2007, 2008, 2010, 2012 Apple Inc. All rights reserved. 3 * Copyright (C) 2007 Justin Haygood (jhaygood (at) reaktix.com) 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of 15 * its contributors may be used to endorse or promote products derived 16 * from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY 19 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 * DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY 22 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #ifndef Atomics_h 31 #define Atomics_h 32 33 #include "wtf/Assertions.h" 34 #include "wtf/CPU.h" 35 36 #include <stdint.h> 37 38 #if COMPILER(MSVC) 39 #include <windows.h> 40 #endif 41 42 #if defined(THREAD_SANITIZER) 43 #include <sanitizer/tsan_interface_atomic.h> 44 #endif 45 46 #if defined(ADDRESS_SANITIZER) 47 #include <sanitizer/asan_interface.h> 48 #endif 49 50 namespace WTF { 51 52 #if COMPILER(MSVC) 53 54 // atomicAdd returns the result of the addition. 55 ALWAYS_INLINE int atomicAdd(int volatile* addend, int increment) 56 { 57 return InterlockedExchangeAdd(reinterpret_cast<long volatile*>(addend), static_cast<long>(increment)) + increment; 58 } 59 60 // atomicSubtract returns the result of the subtraction. 61 ALWAYS_INLINE int atomicSubtract(int volatile* addend, int decrement) 62 { 63 return InterlockedExchangeAdd(reinterpret_cast<long volatile*>(addend), static_cast<long>(-decrement)) - decrement; 64 } 65 66 ALWAYS_INLINE int atomicIncrement(int volatile* addend) { return InterlockedIncrement(reinterpret_cast<long volatile*>(addend)); } 67 ALWAYS_INLINE int atomicDecrement(int volatile* addend) { return InterlockedDecrement(reinterpret_cast<long volatile*>(addend)); } 68 69 ALWAYS_INLINE int64_t atomicIncrement(int64_t volatile* addend) { return InterlockedIncrement64(reinterpret_cast<long long volatile*>(addend)); } 70 ALWAYS_INLINE int64_t atomicDecrement(int64_t volatile* addend) { return InterlockedDecrement64(reinterpret_cast<long long volatile*>(addend)); } 71 72 ALWAYS_INLINE int atomicTestAndSetToOne(int volatile* ptr) 73 { 74 int ret = InterlockedExchange(reinterpret_cast<long volatile*>(ptr), 1); 75 ASSERT(!ret || ret == 1); 76 return ret; 77 } 78 79 ALWAYS_INLINE void atomicSetOneToZero(int volatile* ptr) 80 { 81 ASSERT(*ptr == 1); 82 InterlockedExchange(reinterpret_cast<long volatile*>(ptr), 0); 83 } 84 85 #else 86 87 // atomicAdd returns the result of the addition. 88 ALWAYS_INLINE int atomicAdd(int volatile* addend, int increment) { return __sync_add_and_fetch(addend, increment); } 89 // atomicSubtract returns the result of the subtraction. 90 ALWAYS_INLINE int atomicSubtract(int volatile* addend, int decrement) { return __sync_sub_and_fetch(addend, decrement); } 91 92 ALWAYS_INLINE int atomicIncrement(int volatile* addend) { return atomicAdd(addend, 1); } 93 ALWAYS_INLINE int atomicDecrement(int volatile* addend) { return atomicSubtract(addend, 1); } 94 95 ALWAYS_INLINE int64_t atomicIncrement(int64_t volatile* addend) { return __sync_add_and_fetch(addend, 1); } 96 ALWAYS_INLINE int64_t atomicDecrement(int64_t volatile* addend) { return __sync_sub_and_fetch(addend, 1); } 97 98 ALWAYS_INLINE int atomicTestAndSetToOne(int volatile* ptr) 99 { 100 int ret = __sync_lock_test_and_set(ptr, 1); 101 ASSERT(!ret || ret == 1); 102 return ret; 103 } 104 105 ALWAYS_INLINE void atomicSetOneToZero(int volatile* ptr) 106 { 107 ASSERT(*ptr == 1); 108 __sync_lock_release(ptr); 109 } 110 #endif 111 112 #if defined(THREAD_SANITIZER) 113 114 ALWAYS_INLINE void releaseStore(volatile int* ptr, int value) 115 { 116 __tsan_atomic32_store(ptr, value, __tsan_memory_order_release); 117 } 118 119 ALWAYS_INLINE int acquireLoad(volatile const int* ptr) 120 { 121 return __tsan_atomic32_load(ptr, __tsan_memory_order_acquire); 122 } 123 124 ALWAYS_INLINE void releaseStore(volatile unsigned* ptr, unsigned value) 125 { 126 __tsan_atomic32_store(reinterpret_cast<volatile int*>(ptr), static_cast<int>(value), __tsan_memory_order_release); 127 } 128 129 ALWAYS_INLINE unsigned acquireLoad(volatile const unsigned* ptr) 130 { 131 return static_cast<unsigned>(__tsan_atomic32_load(reinterpret_cast<volatile const int*>(ptr), __tsan_memory_order_acquire)); 132 } 133 134 #else 135 136 #if CPU(X86) || CPU(X86_64) 137 // Only compiler barrier is needed. 138 #if COMPILER(MSVC) 139 // Starting from Visual Studio 2005 compiler guarantees acquire and release 140 // semantics for operations on volatile variables. See MSDN entry for 141 // MemoryBarrier macro. 142 #define MEMORY_BARRIER() 143 #else 144 #define MEMORY_BARRIER() __asm__ __volatile__("" : : : "memory") 145 #endif 146 #elif CPU(ARM) && (OS(LINUX) || OS(ANDROID)) 147 // On ARM __sync_synchronize generates dmb which is very expensive on single 148 // core devices which don't actually need it. Avoid the cost by calling into 149 // kuser_memory_barrier helper. 150 inline void memoryBarrier() 151 { 152 // Note: This is a function call, which is also an implicit compiler barrier. 153 typedef void (*KernelMemoryBarrierFunc)(); 154 ((KernelMemoryBarrierFunc)0xffff0fa0)(); 155 } 156 #define MEMORY_BARRIER() memoryBarrier() 157 #else 158 // Fallback to the compiler intrinsic on all other platforms. 159 #define MEMORY_BARRIER() __sync_synchronize() 160 #endif 161 162 ALWAYS_INLINE void releaseStore(volatile int* ptr, int value) 163 { 164 MEMORY_BARRIER(); 165 *ptr = value; 166 } 167 168 ALWAYS_INLINE int acquireLoad(volatile const int* ptr) 169 { 170 int value = *ptr; 171 MEMORY_BARRIER(); 172 return value; 173 } 174 175 ALWAYS_INLINE void releaseStore(volatile unsigned* ptr, unsigned value) 176 { 177 MEMORY_BARRIER(); 178 *ptr = value; 179 } 180 181 ALWAYS_INLINE unsigned acquireLoad(volatile const unsigned* ptr) 182 { 183 unsigned value = *ptr; 184 MEMORY_BARRIER(); 185 return value; 186 } 187 188 #if defined(ADDRESS_SANITIZER) 189 190 // FIXME: See comment on NO_SANITIZE_ADDRESS in platform/heap/AddressSanitizer.h 191 #if !OS(WIN) || COMPILER(CLANG) 192 #define NO_SANITIZE_ADDRESS_ATOMICS __attribute__((no_sanitize_address)) 193 #else 194 #define NO_SANITIZE_ADDRESS_ATOMICS 195 #endif 196 197 NO_SANITIZE_ADDRESS_ATOMICS ALWAYS_INLINE void asanUnsafeReleaseStore(volatile unsigned* ptr, unsigned value) 198 { 199 MEMORY_BARRIER(); 200 *ptr = value; 201 } 202 203 NO_SANITIZE_ADDRESS_ATOMICS ALWAYS_INLINE unsigned asanUnsafeAcquireLoad(volatile const unsigned* ptr) 204 { 205 unsigned value = *ptr; 206 MEMORY_BARRIER(); 207 return value; 208 } 209 210 #undef NO_SANITIZE_ADDRESS_ATOMICS 211 212 #endif // defined(ADDRESS_SANITIZER) 213 214 #undef MEMORY_BARRIER 215 216 #endif 217 218 #if !defined(ADDRESS_SANITIZER) 219 220 ALWAYS_INLINE void asanUnsafeReleaseStore(volatile unsigned* ptr, unsigned value) 221 { 222 releaseStore(ptr, value); 223 } 224 225 ALWAYS_INLINE unsigned asanUnsafeAcquireLoad(volatile const unsigned* ptr) 226 { 227 return acquireLoad(ptr); 228 } 229 230 #endif 231 232 } // namespace WTF 233 234 using WTF::atomicAdd; 235 using WTF::atomicSubtract; 236 using WTF::atomicDecrement; 237 using WTF::atomicIncrement; 238 using WTF::atomicTestAndSetToOne; 239 using WTF::atomicSetOneToZero; 240 using WTF::acquireLoad; 241 using WTF::releaseStore; 242 243 // These methods allow loading from and storing to poisoned memory. Only 244 // use these methods if you know what you are doing since they will 245 // silence use-after-poison errors from ASan. 246 using WTF::asanUnsafeAcquireLoad; 247 using WTF::asanUnsafeReleaseStore; 248 249 #endif // Atomics_h 250