1 // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 5 // AtomicPointer provides storage for a lock-free pointer. 6 // Platform-dependent implementation of AtomicPointer: 7 // - If the platform provides a cheap barrier, we use it with raw pointers 8 // - If cstdatomic is present (on newer versions of gcc, it is), we use 9 // a cstdatomic-based AtomicPointer. However we prefer the memory 10 // barrier based version, because at least on a gcc 4.4 32-bit build 11 // on linux, we have encountered a buggy <cstdatomic> 12 // implementation. Also, some <cstdatomic> implementations are much 13 // slower than a memory-barrier based implementation (~16ns for 14 // <cstdatomic> based acquire-load vs. ~1ns for a barrier based 15 // acquire-load). 16 // This code is based on atomicops-internals-* in Google's perftools: 17 // http://code.google.com/p/google-perftools/source/browse/#svn%2Ftrunk%2Fsrc%2Fbase 18 19 #ifndef PORT_ATOMIC_POINTER_H_ 20 #define PORT_ATOMIC_POINTER_H_ 21 22 #include <stdint.h> 23 #ifdef LEVELDB_CSTDATOMIC_PRESENT 24 #include <cstdatomic> 25 #endif 26 #ifdef OS_WIN 27 #include <windows.h> 28 #endif 29 #ifdef OS_MACOSX 30 #include <libkern/OSAtomic.h> 31 #endif 32 33 #if defined(_M_X64) || defined(__x86_64__) 34 #define ARCH_CPU_X86_FAMILY 1 35 #elif defined(_M_IX86) || defined(__i386__) || defined(__i386) 36 #define ARCH_CPU_X86_FAMILY 1 37 #elif defined(__ARMEL__) 38 #define ARCH_CPU_ARM_FAMILY 1 39 #elif defined(__ppc__) || defined(__powerpc__) || defined(__powerpc64__) 40 #define ARCH_CPU_PPC_FAMILY 1 41 #endif 42 43 namespace leveldb { 44 namespace port { 45 46 // Define MemoryBarrier() if available 47 // Windows on x86 48 #if defined(OS_WIN) && defined(COMPILER_MSVC) && defined(ARCH_CPU_X86_FAMILY) 49 // windows.h already provides a MemoryBarrier(void) macro 50 // http://msdn.microsoft.com/en-us/library/ms684208(v=vs.85).aspx 51 #define LEVELDB_HAVE_MEMORY_BARRIER 52 53 // Mac OS 54 #elif defined(OS_MACOSX) 55 inline void MemoryBarrier() { 56 OSMemoryBarrier(); 57 } 58 #define LEVELDB_HAVE_MEMORY_BARRIER 59 60 // Gcc on x86 61 #elif defined(ARCH_CPU_X86_FAMILY) && defined(__GNUC__) 62 inline void MemoryBarrier() { 63 // See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on 64 // this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering. 65 __asm__ __volatile__("" : : : "memory"); 66 } 67 #define LEVELDB_HAVE_MEMORY_BARRIER 68 69 // Sun Studio 70 #elif defined(ARCH_CPU_X86_FAMILY) && defined(__SUNPRO_CC) 71 inline void MemoryBarrier() { 72 // See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on 73 // this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering. 74 asm volatile("" : : : "memory"); 75 } 76 #define LEVELDB_HAVE_MEMORY_BARRIER 77 78 // ARM Linux 79 #elif defined(ARCH_CPU_ARM_FAMILY) && defined(__linux__) 80 typedef void (*LinuxKernelMemoryBarrierFunc)(void); 81 // The Linux ARM kernel provides a highly optimized device-specific memory 82 // barrier function at a fixed memory address that is mapped in every 83 // user-level process. 84 // 85 // This beats using CPU-specific instructions which are, on single-core 86 // devices, un-necessary and very costly (e.g. ARMv7-A "dmb" takes more 87 // than 180ns on a Cortex-A8 like the one on a Nexus One). Benchmarking 88 // shows that the extra function call cost is completely negligible on 89 // multi-core devices. 90 // 91 inline void MemoryBarrier() { 92 (*(LinuxKernelMemoryBarrierFunc)0xffff0fa0)(); 93 } 94 #define LEVELDB_HAVE_MEMORY_BARRIER 95 96 // PPC 97 #elif defined(ARCH_CPU_PPC_FAMILY) && defined(__GNUC__) 98 inline void MemoryBarrier() { 99 // TODO for some powerpc expert: is there a cheaper suitable variant? 100 // Perhaps by having separate barriers for acquire and release ops. 101 asm volatile("sync" : : : "memory"); 102 } 103 #define LEVELDB_HAVE_MEMORY_BARRIER 104 105 #endif 106 107 // AtomicPointer built using platform-specific MemoryBarrier() 108 #if defined(LEVELDB_HAVE_MEMORY_BARRIER) 109 class AtomicPointer { 110 private: 111 void* rep_; 112 public: 113 AtomicPointer() { } 114 explicit AtomicPointer(void* p) : rep_(p) {} 115 inline void* NoBarrier_Load() const { return rep_; } 116 inline void NoBarrier_Store(void* v) { rep_ = v; } 117 inline void* Acquire_Load() const { 118 void* result = rep_; 119 MemoryBarrier(); 120 return result; 121 } 122 inline void Release_Store(void* v) { 123 MemoryBarrier(); 124 rep_ = v; 125 } 126 }; 127 128 // AtomicPointer based on <cstdatomic> 129 #elif defined(LEVELDB_CSTDATOMIC_PRESENT) 130 class AtomicPointer { 131 private: 132 std::atomic<void*> rep_; 133 public: 134 AtomicPointer() { } 135 explicit AtomicPointer(void* v) : rep_(v) { } 136 inline void* Acquire_Load() const { 137 return rep_.load(std::memory_order_acquire); 138 } 139 inline void Release_Store(void* v) { 140 rep_.store(v, std::memory_order_release); 141 } 142 inline void* NoBarrier_Load() const { 143 return rep_.load(std::memory_order_relaxed); 144 } 145 inline void NoBarrier_Store(void* v) { 146 rep_.store(v, std::memory_order_relaxed); 147 } 148 }; 149 150 // Atomic pointer based on sparc memory barriers 151 #elif defined(__sparcv9) && defined(__GNUC__) 152 class AtomicPointer { 153 private: 154 void* rep_; 155 public: 156 AtomicPointer() { } 157 explicit AtomicPointer(void* v) : rep_(v) { } 158 inline void* Acquire_Load() const { 159 void* val; 160 __asm__ __volatile__ ( 161 "ldx [%[rep_]], %[val] \n\t" 162 "membar #LoadLoad|#LoadStore \n\t" 163 : [val] "=r" (val) 164 : [rep_] "r" (&rep_) 165 : "memory"); 166 return val; 167 } 168 inline void Release_Store(void* v) { 169 __asm__ __volatile__ ( 170 "membar #LoadStore|#StoreStore \n\t" 171 "stx %[v], [%[rep_]] \n\t" 172 : 173 : [rep_] "r" (&rep_), [v] "r" (v) 174 : "memory"); 175 } 176 inline void* NoBarrier_Load() const { return rep_; } 177 inline void NoBarrier_Store(void* v) { rep_ = v; } 178 }; 179 180 // Atomic pointer based on ia64 acq/rel 181 #elif defined(__ia64) && defined(__GNUC__) 182 class AtomicPointer { 183 private: 184 void* rep_; 185 public: 186 AtomicPointer() { } 187 explicit AtomicPointer(void* v) : rep_(v) { } 188 inline void* Acquire_Load() const { 189 void* val ; 190 __asm__ __volatile__ ( 191 "ld8.acq %[val] = [%[rep_]] \n\t" 192 : [val] "=r" (val) 193 : [rep_] "r" (&rep_) 194 : "memory" 195 ); 196 return val; 197 } 198 inline void Release_Store(void* v) { 199 __asm__ __volatile__ ( 200 "st8.rel [%[rep_]] = %[v] \n\t" 201 : 202 : [rep_] "r" (&rep_), [v] "r" (v) 203 : "memory" 204 ); 205 } 206 inline void* NoBarrier_Load() const { return rep_; } 207 inline void NoBarrier_Store(void* v) { rep_ = v; } 208 }; 209 210 // We have neither MemoryBarrier(), nor <cstdatomic> 211 #else 212 #error Please implement AtomicPointer for this platform. 213 214 #endif 215 216 #undef LEVELDB_HAVE_MEMORY_BARRIER 217 #undef ARCH_CPU_X86_FAMILY 218 #undef ARCH_CPU_ARM_FAMILY 219 #undef ARCH_CPU_PPC_FAMILY 220 221 } // namespace port 222 } // namespace leveldb 223 224 #endif // PORT_ATOMIC_POINTER_H_ 225