1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "runtime/internal/atomic" 9 "unsafe" 10 ) 11 12 // These functions cannot have go:noescape annotations, 13 // because while ptr does not escape, new does. 14 // If new is marked as not escaping, the compiler will make incorrect 15 // escape analysis decisions about the pointer value being stored. 16 // Instead, these are wrappers around the actual atomics (casp1 and so on) 17 // that use noescape to convey which arguments do not escape. 18 19 // atomicstorep performs *ptr = new atomically and invokes a write barrier. 20 // 21 //go:nosplit 22 func atomicstorep(ptr unsafe.Pointer, new unsafe.Pointer) { 23 writebarrierptr_prewrite((*uintptr)(ptr), uintptr(new)) 24 atomic.StorepNoWB(noescape(ptr), new) 25 } 26 27 //go:nosplit 28 func casp(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool { 29 // The write barrier is only necessary if the CAS succeeds, 30 // but since it needs to happen before the write becomes 31 // public, we have to do it conservatively all the time. 32 writebarrierptr_prewrite((*uintptr)(unsafe.Pointer(ptr)), uintptr(new)) 33 return atomic.Casp1((*unsafe.Pointer)(noescape(unsafe.Pointer(ptr))), noescape(old), new) 34 } 35 36 // Like above, but implement in terms of sync/atomic's uintptr operations. 37 // We cannot just call the runtime routines, because the race detector expects 38 // to be able to intercept the sync/atomic forms but not the runtime forms. 39 40 //go:linkname sync_atomic_StoreUintptr sync/atomic.StoreUintptr 41 func sync_atomic_StoreUintptr(ptr *uintptr, new uintptr) 42 43 //go:linkname sync_atomic_StorePointer sync/atomic.StorePointer 44 //go:nosplit 45 func sync_atomic_StorePointer(ptr *unsafe.Pointer, new unsafe.Pointer) { 46 writebarrierptr_prewrite((*uintptr)(unsafe.Pointer(ptr)), uintptr(new)) 47 sync_atomic_StoreUintptr((*uintptr)(unsafe.Pointer(ptr)), uintptr(new)) 48 } 49 50 //go:linkname sync_atomic_SwapUintptr sync/atomic.SwapUintptr 51 func sync_atomic_SwapUintptr(ptr *uintptr, new uintptr) uintptr 52 53 //go:linkname sync_atomic_SwapPointer sync/atomic.SwapPointer 54 //go:nosplit 55 func sync_atomic_SwapPointer(ptr *unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer { 56 writebarrierptr_prewrite((*uintptr)(unsafe.Pointer(ptr)), uintptr(new)) 57 old := unsafe.Pointer(sync_atomic_SwapUintptr((*uintptr)(noescape(unsafe.Pointer(ptr))), uintptr(new))) 58 return old 59 } 60 61 //go:linkname sync_atomic_CompareAndSwapUintptr sync/atomic.CompareAndSwapUintptr 62 func sync_atomic_CompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool 63 64 //go:linkname sync_atomic_CompareAndSwapPointer sync/atomic.CompareAndSwapPointer 65 //go:nosplit 66 func sync_atomic_CompareAndSwapPointer(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool { 67 writebarrierptr_prewrite((*uintptr)(unsafe.Pointer(ptr)), uintptr(new)) 68 return sync_atomic_CompareAndSwapUintptr((*uintptr)(noescape(unsafe.Pointer(ptr))), uintptr(old), uintptr(new)) 69 } 70
