1 /** 2 * Many similar implementations exist. See for example libwsbm 3 * or the linux kernel include/atomic.h 4 * 5 * No copyright claimed on this file. 6 * 7 */ 8 9 #ifndef U_ATOMIC_H 10 #define U_ATOMIC_H 11 12 #include "pipe/p_compiler.h" 13 #include "pipe/p_defines.h" 14 15 /* Favor OS-provided implementations. 16 * 17 * Where no OS-provided implementation is available, fall back to 18 * locally coded assembly, compiler intrinsic or ultimately a 19 * mutex-based implementation. 20 */ 21 #if defined(PIPE_OS_SOLARIS) 22 #define PIPE_ATOMIC_OS_SOLARIS 23 #elif defined(PIPE_CC_MSVC) 24 #define PIPE_ATOMIC_MSVC_INTRINSIC 25 #elif (defined(PIPE_CC_MSVC) && defined(PIPE_ARCH_X86)) 26 #define PIPE_ATOMIC_ASM_MSVC_X86 27 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86)) 28 #define PIPE_ATOMIC_ASM_GCC_X86 29 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86_64)) 30 #define PIPE_ATOMIC_ASM_GCC_X86_64 31 #elif defined(PIPE_CC_GCC) && (PIPE_CC_GCC_VERSION >= 401) 32 #define PIPE_ATOMIC_GCC_INTRINSIC 33 #else 34 #error "Unsupported platform" 35 #endif 36 37 38 #if defined(PIPE_ATOMIC_ASM_GCC_X86_64) 39 #define PIPE_ATOMIC "GCC x86_64 assembly" 40 41 #ifdef __cplusplus 42 extern "C" { 43 #endif 44 45 #define p_atomic_set(_v, _i) (*(_v) = (_i)) 46 #define p_atomic_read(_v) (*(_v)) 47 48 static INLINE boolean 49 p_atomic_dec_zero(int32_t *v) 50 { 51 unsigned char c; 52 53 __asm__ __volatile__("lock; decl %0; sete %1":"+m"(*v), "=qm"(c) 54 ::"memory"); 55 56 return c != 0; 57 } 58 59 static INLINE void 60 p_atomic_inc(int32_t *v) 61 { 62 __asm__ __volatile__("lock; incl %0":"+m"(*v)); 63 } 64 65 static INLINE void 66 p_atomic_dec(int32_t *v) 67 { 68 __asm__ __volatile__("lock; decl %0":"+m"(*v)); 69 } 70 71 static INLINE int32_t 72 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new) 73 { 74 return __sync_val_compare_and_swap(v, old, _new); 75 } 76 77 #ifdef __cplusplus 78 } 79 #endif 80 81 #endif /* PIPE_ATOMIC_ASM_GCC_X86_64 */ 82 83 84 #if defined(PIPE_ATOMIC_ASM_GCC_X86) 85 86 #define PIPE_ATOMIC "GCC x86 assembly" 87 88 #ifdef __cplusplus 89 extern "C" { 90 #endif 91 92 #define p_atomic_set(_v, _i) (*(_v) = (_i)) 93 #define p_atomic_read(_v) (*(_v)) 94 95 static INLINE boolean 96 p_atomic_dec_zero(int32_t *v) 97 { 98 unsigned char c; 99 100 __asm__ __volatile__("lock; decl %0; sete %1":"+m"(*v), "=qm"(c) 101 ::"memory"); 102 103 return c != 0; 104 } 105 106 static INLINE void 107 p_atomic_inc(int32_t *v) 108 { 109 __asm__ __volatile__("lock; incl %0":"+m"(*v)); 110 } 111 112 static INLINE void 113 p_atomic_dec(int32_t *v) 114 { 115 __asm__ __volatile__("lock; decl %0":"+m"(*v)); 116 } 117 118 static INLINE int32_t 119 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new) 120 { 121 return __sync_val_compare_and_swap(v, old, _new); 122 } 123 124 #ifdef __cplusplus 125 } 126 #endif 127 128 #endif 129 130 131 132 /* Implementation using GCC-provided synchronization intrinsics 133 */ 134 #if defined(PIPE_ATOMIC_GCC_INTRINSIC) 135 136 #define PIPE_ATOMIC "GCC Sync Intrinsics" 137 138 #ifdef __cplusplus 139 extern "C" { 140 #endif 141 142 #define p_atomic_set(_v, _i) (*(_v) = (_i)) 143 #define p_atomic_read(_v) (*(_v)) 144 145 static INLINE boolean 146 p_atomic_dec_zero(int32_t *v) 147 { 148 return (__sync_sub_and_fetch(v, 1) == 0); 149 } 150 151 static INLINE void 152 p_atomic_inc(int32_t *v) 153 { 154 (void) __sync_add_and_fetch(v, 1); 155 } 156 157 static INLINE void 158 p_atomic_dec(int32_t *v) 159 { 160 (void) __sync_sub_and_fetch(v, 1); 161 } 162 163 static INLINE int32_t 164 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new) 165 { 166 return __sync_val_compare_and_swap(v, old, _new); 167 } 168 169 #ifdef __cplusplus 170 } 171 #endif 172 173 #endif 174 175 176 177 /* Unlocked version for single threaded environments, such as some 178 * windows kernel modules. 179 */ 180 #if defined(PIPE_ATOMIC_OS_UNLOCKED) 181 182 #define PIPE_ATOMIC "Unlocked" 183 184 #define p_atomic_set(_v, _i) (*(_v) = (_i)) 185 #define p_atomic_read(_v) (*(_v)) 186 #define p_atomic_dec_zero(_v) ((boolean) --(*(_v))) 187 #define p_atomic_inc(_v) ((void) (*(_v))++) 188 #define p_atomic_dec(_v) ((void) (*(_v))--) 189 #define p_atomic_cmpxchg(_v, old, _new) (*(_v) == old ? *(_v) = (_new) : *(_v)) 190 191 #endif 192 193 194 /* Locally coded assembly for MSVC on x86: 195 */ 196 #if defined(PIPE_ATOMIC_ASM_MSVC_X86) 197 198 #define PIPE_ATOMIC "MSVC x86 assembly" 199 200 #ifdef __cplusplus 201 extern "C" { 202 #endif 203 204 #define p_atomic_set(_v, _i) (*(_v) = (_i)) 205 #define p_atomic_read(_v) (*(_v)) 206 207 static INLINE boolean 208 p_atomic_dec_zero(int32_t *v) 209 { 210 unsigned char c; 211 212 __asm { 213 mov eax, [v] 214 lock dec dword ptr [eax] 215 sete byte ptr [c] 216 } 217 218 return c != 0; 219 } 220 221 static INLINE void 222 p_atomic_inc(int32_t *v) 223 { 224 __asm { 225 mov eax, [v] 226 lock inc dword ptr [eax] 227 } 228 } 229 230 static INLINE void 231 p_atomic_dec(int32_t *v) 232 { 233 __asm { 234 mov eax, [v] 235 lock dec dword ptr [eax] 236 } 237 } 238 239 static INLINE int32_t 240 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new) 241 { 242 int32_t orig; 243 244 __asm { 245 mov ecx, [v] 246 mov eax, [old] 247 mov edx, [_new] 248 lock cmpxchg [ecx], edx 249 mov [orig], eax 250 } 251 252 return orig; 253 } 254 255 #ifdef __cplusplus 256 } 257 #endif 258 259 #endif 260 261 262 #if defined(PIPE_ATOMIC_MSVC_INTRINSIC) 263 264 #define PIPE_ATOMIC "MSVC Intrinsics" 265 266 #include <intrin.h> 267 268 #pragma intrinsic(_InterlockedIncrement) 269 #pragma intrinsic(_InterlockedDecrement) 270 #pragma intrinsic(_InterlockedCompareExchange) 271 272 #ifdef __cplusplus 273 extern "C" { 274 #endif 275 276 #define p_atomic_set(_v, _i) (*(_v) = (_i)) 277 #define p_atomic_read(_v) (*(_v)) 278 279 static INLINE boolean 280 p_atomic_dec_zero(int32_t *v) 281 { 282 return _InterlockedDecrement((long *)v) == 0; 283 } 284 285 static INLINE void 286 p_atomic_inc(int32_t *v) 287 { 288 _InterlockedIncrement((long *)v); 289 } 290 291 static INLINE void 292 p_atomic_dec(int32_t *v) 293 { 294 _InterlockedDecrement((long *)v); 295 } 296 297 static INLINE int32_t 298 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new) 299 { 300 return _InterlockedCompareExchange((long *)v, _new, old); 301 } 302 303 #ifdef __cplusplus 304 } 305 #endif 306 307 #endif 308 309 #if defined(PIPE_ATOMIC_OS_SOLARIS) 310 311 #define PIPE_ATOMIC "Solaris OS atomic functions" 312 313 #include <atomic.h> 314 315 #ifdef __cplusplus 316 extern "C" { 317 #endif 318 319 #define p_atomic_set(_v, _i) (*(_v) = (_i)) 320 #define p_atomic_read(_v) (*(_v)) 321 322 static INLINE boolean 323 p_atomic_dec_zero(int32_t *v) 324 { 325 uint32_t n = atomic_dec_32_nv((uint32_t *) v); 326 327 return n != 0; 328 } 329 330 #define p_atomic_inc(_v) atomic_inc_32((uint32_t *) _v) 331 #define p_atomic_dec(_v) atomic_dec_32((uint32_t *) _v) 332 333 #define p_atomic_cmpxchg(_v, _old, _new) \ 334 atomic_cas_32( (uint32_t *) _v, (uint32_t) _old, (uint32_t) _new) 335 336 #ifdef __cplusplus 337 } 338 #endif 339 340 #endif 341 342 343 #ifndef PIPE_ATOMIC 344 #error "No pipe_atomic implementation selected" 345 #endif 346 347 348 349 #endif /* U_ATOMIC_H */ 350
