1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import "unsafe" 8 9 var locktab [57]struct { 10 l mutex 11 pad [_CacheLineSize - unsafe.Sizeof(mutex{})]byte 12 } 13 14 func addrLock(addr *uint64) *mutex { 15 return &locktab[(uintptr(unsafe.Pointer(addr))>>3)%uintptr(len(locktab))].l 16 } 17 18 // Atomic add and return new value. 19 //go:nosplit 20 func xadd(val *uint32, delta int32) uint32 { 21 for { 22 oval := *val 23 nval := oval + uint32(delta) 24 if cas(val, oval, nval) { 25 return nval 26 } 27 } 28 } 29 30 //go:noescape 31 //go:linkname xadduintptr runtime.xadd 32 func xadduintptr(ptr *uintptr, delta uintptr) uintptr 33 34 //go:nosplit 35 func xchg(addr *uint32, v uint32) uint32 { 36 for { 37 old := *addr 38 if cas(addr, old, v) { 39 return old 40 } 41 } 42 } 43 44 //go:nosplit 45 func xchgp1(addr unsafe.Pointer, v unsafe.Pointer) unsafe.Pointer { 46 for { 47 old := *(*unsafe.Pointer)(addr) 48 if casp1((*unsafe.Pointer)(addr), old, v) { 49 return old 50 } 51 } 52 } 53 54 //go:nosplit 55 func xchguintptr(addr *uintptr, v uintptr) uintptr { 56 return uintptr(xchg((*uint32)(unsafe.Pointer(addr)), uint32(v))) 57 } 58 59 //go:nosplit 60 func atomicload(addr *uint32) uint32 { 61 return xadd(addr, 0) 62 } 63 64 //go:nosplit 65 func atomicloadp(addr unsafe.Pointer) unsafe.Pointer { 66 return unsafe.Pointer(uintptr(xadd((*uint32)(addr), 0))) 67 } 68 69 //go:nosplit 70 func atomicstorep1(addr unsafe.Pointer, v unsafe.Pointer) { 71 for { 72 old := *(*unsafe.Pointer)(addr) 73 if casp1((*unsafe.Pointer)(addr), old, v) { 74 return 75 } 76 } 77 } 78 79 //go:nosplit 80 func atomicstore(addr *uint32, v uint32) { 81 for { 82 old := *addr 83 if cas(addr, old, v) { 84 return 85 } 86 } 87 } 88 89 //go:nosplit 90 func cas64(addr *uint64, old, new uint64) bool { 91 var ok bool 92 systemstack(func() { 93 lock(addrLock(addr)) 94 if *addr == old { 95 *addr = new 96 ok = true 97 } 98 unlock(addrLock(addr)) 99 }) 100 return ok 101 } 102 103 //go:nosplit 104 func xadd64(addr *uint64, delta int64) uint64 { 105 var r uint64 106 systemstack(func() { 107 lock(addrLock(addr)) 108 r = *addr + uint64(delta) 109 *addr = r 110 unlock(addrLock(addr)) 111 }) 112 return r 113 } 114 115 //go:nosplit 116 func xchg64(addr *uint64, v uint64) uint64 { 117 var r uint64 118 systemstack(func() { 119 lock(addrLock(addr)) 120 r = *addr 121 *addr = v 122 unlock(addrLock(addr)) 123 }) 124 return r 125 } 126 127 //go:nosplit 128 func atomicload64(addr *uint64) uint64 { 129 var r uint64 130 systemstack(func() { 131 lock(addrLock(addr)) 132 r = *addr 133 unlock(addrLock(addr)) 134 }) 135 return r 136 } 137 138 //go:nosplit 139 func atomicstore64(addr *uint64, v uint64) { 140 systemstack(func() { 141 lock(addrLock(addr)) 142 *addr = v 143 unlock(addrLock(addr)) 144 }) 145 } 146 147 //go:nosplit 148 func atomicor8(addr *uint8, v uint8) { 149 // Align down to 4 bytes and use 32-bit CAS. 150 uaddr := uintptr(unsafe.Pointer(addr)) 151 addr32 := (*uint32)(unsafe.Pointer(uaddr &^ 3)) 152 word := uint32(v) << ((uaddr & 3) * 8) // little endian 153 for { 154 old := *addr32 155 if cas(addr32, old, old|word) { 156 return 157 } 158 } 159 } 160 161 //go:nosplit 162 func atomicand8(addr *uint8, v uint8) { 163 // Align down to 4 bytes and use 32-bit CAS. 164 uaddr := uintptr(unsafe.Pointer(addr)) 165 addr32 := (*uint32)(unsafe.Pointer(uaddr &^ 3)) 166 word := uint32(v) << ((uaddr & 3) * 8) // little endian 167 mask := uint32(0xFF) << ((uaddr & 3) * 8) // little endian 168 word |= ^mask 169 for { 170 old := *addr32 171 if cas(addr32, old, old&word) { 172 return 173 } 174 } 175 } 176
