1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // +build !race 6 7 #include "textflag.h" 8 9 // ARM atomic operations, for use by asm_$(GOOS)_arm.s. 10 11 #define DMB_ISHST_7 \ 12 MOVB runtimegoarm(SB), R11; \ 13 CMP $7, R11; \ 14 BLT 2(PC); \ 15 WORD $0xf57ff05a // dmb ishst 16 17 #define DMB_ISH_7 \ 18 MOVB runtimegoarm(SB), R11; \ 19 CMP $7, R11; \ 20 BLT 2(PC); \ 21 WORD $0xf57ff05b // dmb ish 22 23 TEXT armCompareAndSwapUint32(SB),NOSPLIT,$0-13 24 MOVW addr+0(FP), R1 25 MOVW old+4(FP), R2 26 MOVW new+8(FP), R3 27 casloop: 28 // LDREX and STREX were introduced in ARMv6. 29 LDREX (R1), R0 30 CMP R0, R2 31 BNE casfail 32 DMB_ISHST_7 33 STREX R3, (R1), R0 34 CMP $0, R0 35 BNE casloop 36 MOVW $1, R0 37 DMB_ISH_7 38 MOVBU R0, swapped+12(FP) 39 RET 40 casfail: 41 MOVW $0, R0 42 MOVBU R0, swapped+12(FP) 43 RET 44 45 TEXT armCompareAndSwapUint64(SB),NOSPLIT,$0-21 46 BL fastCheck64<>(SB) 47 MOVW addr+0(FP), R1 48 // make unaligned atomic access panic 49 AND.S $7, R1, R2 50 BEQ 2(PC) 51 MOVW R2, (R2) 52 MOVW old_lo+4(FP), R2 53 MOVW old_hi+8(FP), R3 54 MOVW new_lo+12(FP), R4 55 MOVW new_hi+16(FP), R5 56 cas64loop: 57 // LDREXD and STREXD were introduced in ARMv6k. 58 LDREXD (R1), R6 // loads R6 and R7 59 CMP R2, R6 60 BNE cas64fail 61 CMP R3, R7 62 BNE cas64fail 63 DMB_ISHST_7 64 STREXD R4, (R1), R0 // stores R4 and R5 65 CMP $0, R0 66 BNE cas64loop 67 MOVW $1, R0 68 DMB_ISH_7 69 MOVBU R0, swapped+20(FP) 70 RET 71 cas64fail: 72 MOVW $0, R0 73 MOVBU R0, swapped+20(FP) 74 RET 75 76 TEXT armAddUint32(SB),NOSPLIT,$0-12 77 MOVW addr+0(FP), R1 78 MOVW delta+4(FP), R2 79 addloop: 80 // LDREX and STREX were introduced in ARMv6. 81 LDREX (R1), R3 82 ADD R2, R3 83 DMB_ISHST_7 84 STREX R3, (R1), R0 85 CMP $0, R0 86 BNE addloop 87 DMB_ISH_7 88 MOVW R3, new+8(FP) 89 RET 90 91 TEXT armAddUint64(SB),NOSPLIT,$0-20 92 BL fastCheck64<>(SB) 93 MOVW addr+0(FP), R1 94 // make unaligned atomic access panic 95 AND.S $7, R1, R2 96 BEQ 2(PC) 97 MOVW R2, (R2) 98 MOVW delta_lo+4(FP), R2 99 MOVW delta_hi+8(FP), R3 100 add64loop: 101 // LDREXD and STREXD were introduced in ARMv6k. 102 LDREXD (R1), R4 // loads R4 and R5 103 ADD.S R2, R4 104 ADC R3, R5 105 DMB_ISHST_7 106 STREXD R4, (R1), R0 // stores R4 and R5 107 CMP $0, R0 108 BNE add64loop 109 DMB_ISH_7 110 MOVW R4, new_lo+12(FP) 111 MOVW R5, new_hi+16(FP) 112 RET 113 114 TEXT armSwapUint32(SB),NOSPLIT,$0-12 115 MOVW addr+0(FP), R1 116 MOVW new+4(FP), R2 117 swaploop: 118 // LDREX and STREX were introduced in ARMv6. 119 LDREX (R1), R3 120 DMB_ISHST_7 121 STREX R2, (R1), R0 122 CMP $0, R0 123 BNE swaploop 124 DMB_ISH_7 125 MOVW R3, old+8(FP) 126 RET 127 128 TEXT armSwapUint64(SB),NOSPLIT,$0-20 129 BL fastCheck64<>(SB) 130 MOVW addr+0(FP), R1 131 // make unaligned atomic access panic 132 AND.S $7, R1, R2 133 BEQ 2(PC) 134 MOVW R2, (R2) 135 MOVW new_lo+4(FP), R2 136 MOVW new_hi+8(FP), R3 137 swap64loop: 138 // LDREXD and STREXD were introduced in ARMv6k. 139 LDREXD (R1), R4 // loads R4 and R5 140 DMB_ISHST_7 141 STREXD R2, (R1), R0 // stores R2 and R3 142 CMP $0, R0 143 BNE swap64loop 144 DMB_ISH_7 145 MOVW R4, old_lo+12(FP) 146 MOVW R5, old_hi+16(FP) 147 RET 148 149 TEXT armLoadUint64(SB),NOSPLIT,$0-12 150 BL fastCheck64<>(SB) 151 MOVW addr+0(FP), R1 152 // make unaligned atomic access panic 153 AND.S $7, R1, R2 154 BEQ 2(PC) 155 MOVW R2, (R2) 156 load64loop: 157 LDREXD (R1), R2 // loads R2 and R3 158 DMB_ISHST_7 159 STREXD R2, (R1), R0 // stores R2 and R3 160 CMP $0, R0 161 BNE load64loop 162 DMB_ISH_7 163 MOVW R2, val_lo+4(FP) 164 MOVW R3, val_hi+8(FP) 165 RET 166 167 TEXT armStoreUint64(SB),NOSPLIT,$0-12 168 BL fastCheck64<>(SB) 169 MOVW addr+0(FP), R1 170 // make unaligned atomic access panic 171 AND.S $7, R1, R2 172 BEQ 2(PC) 173 MOVW R2, (R2) 174 MOVW val_lo+4(FP), R2 175 MOVW val_hi+8(FP), R3 176 store64loop: 177 LDREXD (R1), R4 // loads R4 and R5 178 DMB_ISHST_7 179 STREXD R2, (R1), R0 // stores R2 and R3 180 CMP $0, R0 181 BNE store64loop 182 DMB_ISH_7 183 RET 184 185 // Check for broken 64-bit LDREXD as found in QEMU. 186 // LDREXD followed by immediate STREXD should succeed. 187 // If it fails, try a few times just to be sure (maybe our thread got 188 // rescheduled between the two instructions) and then panic. 189 // A bug in some copies of QEMU makes STREXD never succeed, 190 // which will make uses of the 64-bit atomic operations loop forever. 191 // If things are working, set okLDREXD to avoid future checks. 192 // https://bugs.launchpad.net/qemu/+bug/670883. 193 TEXT check64<>(SB),NOSPLIT,$16-0 194 MOVW $10, R1 195 // 8-aligned stack address scratch space. 196 MOVW $8(R13), R5 197 AND $~7, R5 198 loop: 199 LDREXD (R5), R2 200 STREXD R2, (R5), R0 201 CMP $0, R0 202 BEQ ok 203 SUB $1, R1 204 CMP $0, R1 205 BNE loop 206 // Must be buggy QEMU. 207 BL panic64(SB) 208 ok: 209 RET 210 211 // Fast, cached version of check. No frame, just MOVW CMP RET after first time. 212 TEXT fastCheck64<>(SB),NOSPLIT,$-4 213 MOVW ok64<>(SB), R0 214 CMP $0, R0 // have we been here before? 215 RET.NE 216 B slowCheck64<>(SB) 217 218 TEXT slowCheck64<>(SB),NOSPLIT,$0-0 219 BL check64<>(SB) 220 // Still here, must be okay. 221 MOVW $1, R0 222 MOVW R0, ok64<>(SB) 223 RET 224 225 GLOBL ok64<>(SB), NOPTR, $4 226
