1 /* 2 * Copyright (C) 2007, 2008, 2010, 2012 Apple Inc. All rights reserved. 3 * Copyright (C) 2007 Justin Haygood (jhaygood (at) reaktix.com) 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of 15 * its contributors may be used to endorse or promote products derived 16 * from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY 19 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 * DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY 22 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #ifndef Atomics_h 31 #define Atomics_h 32 33 #include "wtf/Assertions.h" 34 #include "wtf/CPU.h" 35 36 #include <stdint.h> 37 38 #if COMPILER(MSVC) 39 #include <windows.h> 40 #endif 41 42 #if defined(THREAD_SANITIZER) 43 #include <sanitizer/tsan_interface_atomic.h> 44 #endif 45 46 namespace WTF { 47 48 #if COMPILER(MSVC) 49 50 // atomicAdd returns the result of the addition. 51 ALWAYS_INLINE int atomicAdd(int volatile* addend, int increment) 52 { 53 return InterlockedExchangeAdd(reinterpret_cast<long volatile*>(addend), static_cast<long>(increment)) + increment; 54 } 55 56 // atomicSubtract returns the result of the subtraction. 57 ALWAYS_INLINE int atomicSubtract(int volatile* addend, int decrement) 58 { 59 return InterlockedExchangeAdd(reinterpret_cast<long volatile*>(addend), static_cast<long>(-decrement)) - decrement; 60 } 61 62 ALWAYS_INLINE int atomicIncrement(int volatile* addend) { return InterlockedIncrement(reinterpret_cast<long volatile*>(addend)); } 63 ALWAYS_INLINE int atomicDecrement(int volatile* addend) { return InterlockedDecrement(reinterpret_cast<long volatile*>(addend)); } 64 65 ALWAYS_INLINE int64_t atomicIncrement(int64_t volatile* addend) { return InterlockedIncrement64(reinterpret_cast<long long volatile*>(addend)); } 66 ALWAYS_INLINE int64_t atomicDecrement(int64_t volatile* addend) { return InterlockedDecrement64(reinterpret_cast<long long volatile*>(addend)); } 67 68 ALWAYS_INLINE int atomicTestAndSetToOne(int volatile* ptr) 69 { 70 int ret = InterlockedExchange(reinterpret_cast<long volatile*>(ptr), 1); 71 ASSERT(!ret || ret == 1); 72 return ret; 73 } 74 75 ALWAYS_INLINE void atomicSetOneToZero(int volatile* ptr) 76 { 77 ASSERT(*ptr == 1); 78 InterlockedExchange(reinterpret_cast<long volatile*>(ptr), 0); 79 } 80 81 #else 82 83 // atomicAdd returns the result of the addition. 84 ALWAYS_INLINE int atomicAdd(int volatile* addend, int increment) { return __sync_add_and_fetch(addend, increment); } 85 // atomicSubtract returns the result of the subtraction. 86 ALWAYS_INLINE int atomicSubtract(int volatile* addend, int decrement) { return __sync_sub_and_fetch(addend, decrement); } 87 88 ALWAYS_INLINE int atomicIncrement(int volatile* addend) { return atomicAdd(addend, 1); } 89 ALWAYS_INLINE int atomicDecrement(int volatile* addend) { return atomicSubtract(addend, 1); } 90 91 ALWAYS_INLINE int64_t atomicIncrement(int64_t volatile* addend) { return __sync_add_and_fetch(addend, 1); } 92 ALWAYS_INLINE int64_t atomicDecrement(int64_t volatile* addend) { return __sync_sub_and_fetch(addend, 1); } 93 94 ALWAYS_INLINE int atomicTestAndSetToOne(int volatile* ptr) 95 { 96 int ret = __sync_lock_test_and_set(ptr, 1); 97 ASSERT(!ret || ret == 1); 98 return ret; 99 } 100 101 ALWAYS_INLINE void atomicSetOneToZero(int volatile* ptr) 102 { 103 ASSERT(*ptr == 1); 104 __sync_lock_release(ptr); 105 } 106 #endif 107 108 #if defined(THREAD_SANITIZER) 109 ALWAYS_INLINE void releaseStore(volatile int* ptr, int value) 110 { 111 __tsan_atomic32_store(ptr, value, __tsan_memory_order_release); 112 } 113 114 ALWAYS_INLINE int acquireLoad(volatile const int* ptr) 115 { 116 return __tsan_atomic32_load(ptr, __tsan_memory_order_acquire); 117 } 118 #else 119 120 #if CPU(X86) || CPU(X86_64) 121 // Only compiler barrier is needed. 122 #if COMPILER(MSVC) 123 // Starting from Visual Studio 2005 compiler guarantees acquire and release 124 // semantics for operations on volatile variables. See MSDN entry for 125 // MemoryBarrier macro. 126 #define MEMORY_BARRIER() 127 #else 128 #define MEMORY_BARRIER() __asm__ __volatile__("" : : : "memory") 129 #endif 130 #elif CPU(ARM) && (OS(LINUX) || OS(ANDROID)) 131 // On ARM __sync_synchronize generates dmb which is very expensive on single 132 // core devices which don't actually need it. Avoid the cost by calling into 133 // kuser_memory_barrier helper. 134 inline void memoryBarrier() 135 { 136 // Note: This is a function call, which is also an implicit compiler barrier. 137 typedef void (*KernelMemoryBarrierFunc)(); 138 ((KernelMemoryBarrierFunc)0xffff0fa0)(); 139 } 140 #define MEMORY_BARRIER() memoryBarrier() 141 #else 142 // Fallback to the compiler intrinsic on all other platforms. 143 #define MEMORY_BARRIER() __sync_synchronize() 144 #endif 145 146 ALWAYS_INLINE void releaseStore(volatile int* ptr, int value) 147 { 148 MEMORY_BARRIER(); 149 *ptr = value; 150 } 151 152 ALWAYS_INLINE int acquireLoad(volatile const int* ptr) 153 { 154 int value = *ptr; 155 MEMORY_BARRIER(); 156 return value; 157 } 158 159 #undef MEMORY_BARRIER 160 161 #endif 162 163 } // namespace WTF 164 165 using WTF::atomicAdd; 166 using WTF::atomicSubtract; 167 using WTF::atomicDecrement; 168 using WTF::atomicIncrement; 169 using WTF::atomicTestAndSetToOne; 170 using WTF::atomicSetOneToZero; 171 using WTF::acquireLoad; 172 using WTF::releaseStore; 173 174 #endif // Atomics_h 175