1 /*- 2 * Copyright (c) 2009-2010 Brad Penoff 3 * Copyright (c) 2009-2010 Humaira Kamal 4 * Copyright (c) 2011-2012 Irene Ruengeler 5 * Copyright (c) 2011-2012 Michael Tuexen 6 * 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #ifndef _USER_ATOMIC_H_ 32 #define _USER_ATOMIC_H_ 33 34 /* __Userspace__ version of sys/i386/include/atomic.h goes here */ 35 36 /* TODO In the future, might want to not use i386 specific assembly. 37 * The options include: 38 * - implement them generically (but maybe not truly atomic?) in userspace 39 * - have ifdef's for __Userspace_arch_ perhaps (OS isn't enough...) 40 */ 41 42 #include <stdio.h> 43 #include <sys/types.h> 44 45 #if defined(__Userspace_os_Darwin) || defined (__Userspace_os_Windows) 46 #if defined (__Userspace_os_Windows) 47 #define atomic_add_int(addr, val) InterlockedExchangeAdd((LPLONG)addr, (LONG)val) 48 #define atomic_fetchadd_int(addr, val) InterlockedExchangeAdd((LPLONG)addr, (LONG)val) 49 #define atomic_subtract_int(addr, val) InterlockedExchangeAdd((LPLONG)addr,-((LONG)val)) 50 #define atomic_cmpset_int(dst, exp, src) InterlockedCompareExchange((LPLONG)dst, src, exp) 51 #define SCTP_DECREMENT_AND_CHECK_REFCOUNT(addr) (InterlockedExchangeAdd((LPLONG)addr, (-1L)) == 1) 52 #else 53 #include <libkern/OSAtomic.h> 54 #define atomic_add_int(addr, val) OSAtomicAdd32Barrier(val, (int32_t *)addr) 55 #define atomic_fetchadd_int(addr, val) OSAtomicAdd32Barrier(val, (int32_t *)addr) 56 #define atomic_subtract_int(addr, val) OSAtomicAdd32Barrier(-val, (int32_t *)addr) 57 #define atomic_cmpset_int(dst, exp, src) OSAtomicCompareAndSwapIntBarrier(exp, src, (int *)dst) 58 #define SCTP_DECREMENT_AND_CHECK_REFCOUNT(addr) (atomic_fetchadd_int(addr, -1) == 0) 59 #endif 60 61 #if defined(INVARIANTS) 62 #define SCTP_SAVE_ATOMIC_DECREMENT(addr, val) \ 63 { \ 64 int32_t newval; \ 65 newval = atomic_fetchadd_int(addr, -val); \ 66 if (newval < 0) { \ 67 panic("Counter goes negative"); \ 68 } \ 69 } 70 #else 71 #define SCTP_SAVE_ATOMIC_DECREMENT(addr, val) \ 72 { \ 73 int32_t newval; \ 74 newval = atomic_fetchadd_int(addr, -val); \ 75 if (newval < 0) { \ 76 *addr = 0; \ 77 } \ 78 } 79 #if defined(__Userspace_os_Windows) 80 static void atomic_init() {} /* empty when we are not using atomic_mtx */ 81 #else 82 static inline void atomic_init() {} /* empty when we are not using atomic_mtx */ 83 #endif 84 #endif 85 86 #else 87 /* Using gcc built-in functions for atomic memory operations 88 Reference: http://gcc.gnu.org/onlinedocs/gcc-4.1.0/gcc/Atomic-Builtins.html 89 Requires gcc version 4.1.0 90 compile with -march=i486 91 */ 92 93 /*Atomically add V to *P.*/ 94 #define atomic_add_int(P, V) (void) __sync_fetch_and_add(P, V) 95 96 /*Atomically subtrace V from *P.*/ 97 #define atomic_subtract_int(P, V) (void) __sync_fetch_and_sub(P, V) 98 99 /* 100 * Atomically add the value of v to the integer pointed to by p and return 101 * the previous value of *p. 102 */ 103 #define atomic_fetchadd_int(p, v) __sync_fetch_and_add(p, v) 104 105 /* Following explanation from src/sys/i386/include/atomic.h, 106 * for atomic compare and set 107 * 108 * if (*dst == exp) *dst = src (all 32 bit words) 109 * 110 * Returns 0 on failure, non-zero on success 111 */ 112 113 #define atomic_cmpset_int(dst, exp, src) __sync_bool_compare_and_swap(dst, exp, src) 114 115 #define SCTP_DECREMENT_AND_CHECK_REFCOUNT(addr) (atomic_fetchadd_int(addr, -1) == 1) 116 #if defined(INVARIANTS) 117 #define SCTP_SAVE_ATOMIC_DECREMENT(addr, val) \ 118 { \ 119 int32_t oldval; \ 120 oldval = atomic_fetchadd_int(addr, -val); \ 121 if (oldval < val) { \ 122 panic("Counter goes negative"); \ 123 } \ 124 } 125 #else 126 #define SCTP_SAVE_ATOMIC_DECREMENT(addr, val) \ 127 { \ 128 int32_t oldval; \ 129 oldval = atomic_fetchadd_int(addr, -val); \ 130 if (oldval < val) { \ 131 *addr = 0; \ 132 } \ 133 } 134 #endif 135 static inline void atomic_init() {} /* empty when we are not using atomic_mtx */ 136 #endif 137 138 #if 0 /* using libatomic_ops */ 139 #include "user_include/atomic_ops.h" 140 141 /*Atomically add incr to *P, and return the original value of *P.*/ 142 #define atomic_add_int(P, V) AO_fetch_and_add((AO_t*)P, V) 143 144 #define atomic_subtract_int(P, V) AO_fetch_and_add((AO_t*)P, -(V)) 145 146 /* 147 * Atomically add the value of v to the integer pointed to by p and return 148 * the previous value of *p. 149 */ 150 #define atomic_fetchadd_int(p, v) AO_fetch_and_add((AO_t*)p, v) 151 152 /* Atomically compare *addr to old_val, and replace *addr by new_val 153 if the first comparison succeeds. Returns nonzero if the comparison 154 succeeded and *addr was updated. 155 */ 156 /* Following Explanation from src/sys/i386/include/atomic.h, which 157 matches that of AO_compare_and_swap above. 158 * Atomic compare and set, used by the mutex functions 159 * 160 * if (*dst == exp) *dst = src (all 32 bit words) 161 * 162 * Returns 0 on failure, non-zero on success 163 */ 164 165 #define atomic_cmpset_int(dst, exp, src) AO_compare_and_swap((AO_t*)dst, exp, src) 166 167 static inline void atomic_init() {} /* empty when we are not using atomic_mtx */ 168 #endif /* closing #if for libatomic */ 169 170 #if 0 /* using atomic_mtx */ 171 172 #include <pthread.h> 173 174 extern userland_mutex_t atomic_mtx; 175 176 #if defined (__Userspace_os_Windows) 177 static inline void atomic_init() { 178 InitializeCriticalSection(&atomic_mtx); 179 } 180 static inline void atomic_destroy() { 181 DeleteCriticalSection(&atomic_mtx); 182 } 183 static inline void atomic_lock() { 184 EnterCriticalSection(&atomic_mtx); 185 } 186 static inline void atomic_unlock() { 187 LeaveCriticalSection(&atomic_mtx); 188 } 189 #else 190 static inline void atomic_init() { 191 (void)pthread_mutex_init(&atomic_mtx, NULL); 192 } 193 static inline void atomic_destroy() { 194 (void)pthread_mutex_destroy(&atomic_mtx); 195 } 196 static inline void atomic_lock() { 197 (void)pthread_mutex_lock(&atomic_mtx); 198 } 199 static inline void atomic_unlock() { 200 (void)pthread_mutex_unlock(&atomic_mtx); 201 } 202 #endif 203 /* 204 * For userland, always use lock prefixes so that the binaries will run 205 * on both SMP and !SMP systems. 206 */ 207 208 #define MPLOCKED "lock ; " 209 210 /* 211 * Atomically add the value of v to the integer pointed to by p and return 212 * the previous value of *p. 213 */ 214 static __inline u_int 215 atomic_fetchadd_int(volatile void *n, u_int v) 216 { 217 int *p = (int *) n; 218 atomic_lock(); 219 __asm __volatile( 220 " " MPLOCKED " " 221 " xaddl %0, %1 ; " 222 "# atomic_fetchadd_int" 223 : "+r" (v), /* 0 (result) */ 224 "=m" (*p) /* 1 */ 225 : "m" (*p)); /* 2 */ 226 atomic_unlock(); 227 228 return (v); 229 } 230 231 232 #ifdef CPU_DISABLE_CMPXCHG 233 234 static __inline int 235 atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src) 236 { 237 u_char res; 238 239 atomic_lock(); 240 __asm __volatile( 241 " pushfl ; " 242 " cli ; " 243 " cmpl %3,%4 ; " 244 " jne 1f ; " 245 " movl %2,%1 ; " 246 "1: " 247 " sete %0 ; " 248 " popfl ; " 249 "# atomic_cmpset_int" 250 : "=q" (res), /* 0 */ 251 "=m" (*dst) /* 1 */ 252 : "r" (src), /* 2 */ 253 "r" (exp), /* 3 */ 254 "m" (*dst) /* 4 */ 255 : "memory"); 256 atomic_unlock(); 257 258 return (res); 259 } 260 261 #else /* !CPU_DISABLE_CMPXCHG */ 262 263 static __inline int 264 atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src) 265 { 266 atomic_lock(); 267 u_char res; 268 269 __asm __volatile( 270 " " MPLOCKED " " 271 " cmpxchgl %2,%1 ; " 272 " sete %0 ; " 273 "1: " 274 "# atomic_cmpset_int" 275 : "=a" (res), /* 0 */ 276 "=m" (*dst) /* 1 */ 277 : "r" (src), /* 2 */ 278 "a" (exp), /* 3 */ 279 "m" (*dst) /* 4 */ 280 : "memory"); 281 atomic_unlock(); 282 283 return (res); 284 } 285 286 #endif /* CPU_DISABLE_CMPXCHG */ 287 288 #define atomic_add_int(P, V) do { \ 289 atomic_lock(); \ 290 (*(u_int *)(P) += (V)); \ 291 atomic_unlock(); \ 292 } while(0) 293 #define atomic_subtract_int(P, V) do { \ 294 atomic_lock(); \ 295 (*(u_int *)(P) -= (V)); \ 296 atomic_unlock(); \ 297 } while(0) 298 299 #endif 300 #endif 301
