1 ; RUN: llc < %s -mtriple=arm64-eabi -asm-verbose=false -verify-machineinstrs -mcpu=cyclone | FileCheck -enable-var-scope %s 2 3 define i32 @val_compare_and_swap(i32* %p, i32 %cmp, i32 %new) #0 { 4 ; CHECK-LABEL: val_compare_and_swap: 5 ; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x0 6 ; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: 7 ; CHECK-NEXT: ldaxr [[RESULT:w[0-9]+]], [x[[ADDR]]] 8 ; CHECK-NEXT: cmp [[RESULT]], w1 9 ; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]] 10 ; CHECK-NEXT: stxr [[SCRATCH_REG:w[0-9]+]], w2, [x[[ADDR]]] 11 ; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]] 12 ; CHECK-NEXT: ret 13 ; CHECK-NEXT: [[FAILBB]]: 14 ; CHECK-NEXT: clrex 15 ; CHECK-NEXT: ret 16 %pair = cmpxchg i32* %p, i32 %cmp, i32 %new acquire acquire 17 %val = extractvalue { i32, i1 } %pair, 0 18 ret i32 %val 19 } 20 21 define i32 @val_compare_and_swap_from_load(i32* %p, i32 %cmp, i32* %pnew) #0 { 22 ; CHECK-LABEL: val_compare_and_swap_from_load: 23 ; CHECK-NEXT: ldr [[NEW:w[0-9]+]], [x2] 24 ; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: 25 ; CHECK-NEXT: ldaxr w[[RESULT:[0-9]+]], [x0] 26 ; CHECK-NEXT: cmp w[[RESULT]], w1 27 ; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]] 28 ; CHECK-NEXT: stxr [[SCRATCH_REG:w[0-9]+]], [[NEW]], [x0] 29 ; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]] 30 ; CHECK-NEXT: mov x0, x[[RESULT]] 31 ; CHECK-NEXT: ret 32 ; CHECK-NEXT: [[FAILBB]]: 33 ; CHECK-NEXT: clrex 34 ; CHECK-NEXT: mov x0, x[[RESULT]] 35 ; CHECK-NEXT: ret 36 %new = load i32, i32* %pnew 37 %pair = cmpxchg i32* %p, i32 %cmp, i32 %new acquire acquire 38 %val = extractvalue { i32, i1 } %pair, 0 39 ret i32 %val 40 } 41 42 define i32 @val_compare_and_swap_rel(i32* %p, i32 %cmp, i32 %new) #0 { 43 ; CHECK-LABEL: val_compare_and_swap_rel: 44 ; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x0 45 ; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: 46 ; CHECK-NEXT: ldaxr [[RESULT:w[0-9]+]], [x[[ADDR]]] 47 ; CHECK-NEXT: cmp [[RESULT]], w1 48 ; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]] 49 ; CHECK-NEXT: stlxr [[SCRATCH_REG:w[0-9]+]], w2, [x[[ADDR]]] 50 ; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]] 51 ; CHECK-NEXT: ret 52 ; CHECK-NEXT: [[FAILBB]]: 53 ; CHECK-NEXT: clrex 54 ; CHECK-NEXT: ret 55 %pair = cmpxchg i32* %p, i32 %cmp, i32 %new acq_rel monotonic 56 %val = extractvalue { i32, i1 } %pair, 0 57 ret i32 %val 58 } 59 60 define i64 @val_compare_and_swap_64(i64* %p, i64 %cmp, i64 %new) #0 { 61 ; CHECK-LABEL: val_compare_and_swap_64: 62 ; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x0 63 ; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: 64 ; CHECK-NEXT: ldxr [[RESULT:x[0-9]+]], [x[[ADDR]]] 65 ; CHECK-NEXT: cmp [[RESULT]], x1 66 ; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]] 67 ; CHECK-NEXT: stxr [[SCRATCH_REG:w[0-9]+]], x2, [x[[ADDR]]] 68 ; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]] 69 ; CHECK-NEXT: ret 70 ; CHECK-NEXT: [[FAILBB]]: 71 ; CHECK-NEXT: clrex 72 ; CHECK-NEXT: ret 73 %pair = cmpxchg i64* %p, i64 %cmp, i64 %new monotonic monotonic 74 %val = extractvalue { i64, i1 } %pair, 0 75 ret i64 %val 76 } 77 78 define i32 @fetch_and_nand(i32* %p) #0 { 79 ; CHECK-LABEL: fetch_and_nand: 80 ; CHECK: [[TRYBB:.?LBB[0-9_]+]]: 81 ; CHECK: ldxr w[[DEST_REG:[0-9]+]], [x0] 82 ; CHECK: mvn [[TMP_REG:w[0-9]+]], w[[DEST_REG]] 83 ; CHECK: orr [[SCRATCH2_REG:w[0-9]+]], [[TMP_REG]], #0xfffffff8 84 ; CHECK-NOT: stlxr [[SCRATCH2_REG]], [[SCRATCH2_REG]] 85 ; CHECK: stlxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x0] 86 ; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]] 87 ; CHECK: mov x0, x[[DEST_REG]] 88 %val = atomicrmw nand i32* %p, i32 7 release 89 ret i32 %val 90 } 91 92 define i64 @fetch_and_nand_64(i64* %p) #0 { 93 ; CHECK-LABEL: fetch_and_nand_64: 94 ; CHECK: mov x[[ADDR:[0-9]+]], x0 95 ; CHECK: [[TRYBB:.?LBB[0-9_]+]]: 96 ; CHECK: ldaxr x[[DEST_REG:[0-9]+]], [x[[ADDR]]] 97 ; CHECK: mvn w[[TMP_REG:[0-9]+]], w[[DEST_REG]] 98 ; CHECK: orr [[SCRATCH2_REG:x[0-9]+]], x[[TMP_REG]], #0xfffffffffffffff8 99 ; CHECK: stlxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x[[ADDR]]] 100 ; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]] 101 102 %val = atomicrmw nand i64* %p, i64 7 acq_rel 103 ret i64 %val 104 } 105 106 define i32 @fetch_and_or(i32* %p) #0 { 107 ; CHECK-LABEL: fetch_and_or: 108 ; CHECK: mov [[OLDVAL_REG:w[0-9]+]], #5 109 ; CHECK: [[TRYBB:.?LBB[0-9_]+]]: 110 ; CHECK: ldaxr w[[DEST_REG:[0-9]+]], [x0] 111 ; CHECK: orr [[SCRATCH2_REG:w[0-9]+]], w[[DEST_REG]], [[OLDVAL_REG]] 112 ; CHECK-NOT: stlxr [[SCRATCH2_REG]], [[SCRATCH2_REG]] 113 ; CHECK: stlxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x0] 114 ; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]] 115 ; CHECK: mov x0, x[[DEST_REG]] 116 %val = atomicrmw or i32* %p, i32 5 seq_cst 117 ret i32 %val 118 } 119 120 define i64 @fetch_and_or_64(i64* %p) #0 { 121 ; CHECK: fetch_and_or_64: 122 ; CHECK: mov x[[ADDR:[0-9]+]], x0 123 ; CHECK: [[TRYBB:.?LBB[0-9_]+]]: 124 ; CHECK: ldxr [[DEST_REG:x[0-9]+]], [x[[ADDR]]] 125 ; CHECK: orr [[SCRATCH2_REG:x[0-9]+]], [[DEST_REG]], #0x7 126 ; CHECK: stxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x[[ADDR]]] 127 ; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]] 128 %val = atomicrmw or i64* %p, i64 7 monotonic 129 ret i64 %val 130 } 131 132 define void @acquire_fence() #0 { 133 fence acquire 134 ret void 135 ; CHECK-LABEL: acquire_fence: 136 ; CHECK: dmb ishld 137 } 138 139 define void @release_fence() #0 { 140 fence release 141 ret void 142 ; CHECK-LABEL: release_fence: 143 ; CHECK: dmb ish{{$}} 144 } 145 146 define void @seq_cst_fence() #0 { 147 fence seq_cst 148 ret void 149 ; CHECK-LABEL: seq_cst_fence: 150 ; CHECK: dmb ish{{$}} 151 } 152 153 define i32 @atomic_load(i32* %p) #0 { 154 %r = load atomic i32, i32* %p seq_cst, align 4 155 ret i32 %r 156 ; CHECK-LABEL: atomic_load: 157 ; CHECK: ldar 158 } 159 160 define i8 @atomic_load_relaxed_8(i8* %p, i32 %off32) #0 { 161 ; CHECK-LABEL: atomic_load_relaxed_8: 162 %ptr_unsigned = getelementptr i8, i8* %p, i32 4095 163 %val_unsigned = load atomic i8, i8* %ptr_unsigned monotonic, align 1 164 ; CHECK: ldrb {{w[0-9]+}}, [x0, #4095] 165 166 %ptr_regoff = getelementptr i8, i8* %p, i32 %off32 167 %val_regoff = load atomic i8, i8* %ptr_regoff unordered, align 1 168 %tot1 = add i8 %val_unsigned, %val_regoff 169 ; CHECK: ldrb {{w[0-9]+}}, [x0, w1, sxtw] 170 171 %ptr_unscaled = getelementptr i8, i8* %p, i32 -256 172 %val_unscaled = load atomic i8, i8* %ptr_unscaled monotonic, align 1 173 %tot2 = add i8 %tot1, %val_unscaled 174 ; CHECK: ldurb {{w[0-9]+}}, [x0, #-256] 175 176 %ptr_random = getelementptr i8, i8* %p, i32 1191936 ; 0x123000 (i.e. ADD imm) 177 %val_random = load atomic i8, i8* %ptr_random unordered, align 1 178 %tot3 = add i8 %tot2, %val_random 179 ; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 180 ; CHECK: ldrb {{w[0-9]+}}, [x[[ADDR]]] 181 182 ret i8 %tot3 183 } 184 185 define i16 @atomic_load_relaxed_16(i16* %p, i32 %off32) #0 { 186 ; CHECK-LABEL: atomic_load_relaxed_16: 187 %ptr_unsigned = getelementptr i16, i16* %p, i32 4095 188 %val_unsigned = load atomic i16, i16* %ptr_unsigned monotonic, align 2 189 ; CHECK: ldrh {{w[0-9]+}}, [x0, #8190] 190 191 %ptr_regoff = getelementptr i16, i16* %p, i32 %off32 192 %val_regoff = load atomic i16, i16* %ptr_regoff unordered, align 2 193 %tot1 = add i16 %val_unsigned, %val_regoff 194 ; CHECK: ldrh {{w[0-9]+}}, [x0, w1, sxtw #1] 195 196 %ptr_unscaled = getelementptr i16, i16* %p, i32 -128 197 %val_unscaled = load atomic i16, i16* %ptr_unscaled monotonic, align 2 198 %tot2 = add i16 %tot1, %val_unscaled 199 ; CHECK: ldurh {{w[0-9]+}}, [x0, #-256] 200 201 %ptr_random = getelementptr i16, i16* %p, i32 595968 ; 0x123000/2 (i.e. ADD imm) 202 %val_random = load atomic i16, i16* %ptr_random unordered, align 2 203 %tot3 = add i16 %tot2, %val_random 204 ; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 205 ; CHECK: ldrh {{w[0-9]+}}, [x[[ADDR]]] 206 207 ret i16 %tot3 208 } 209 210 define i32 @atomic_load_relaxed_32(i32* %p, i32 %off32) #0 { 211 ; CHECK-LABEL: atomic_load_relaxed_32: 212 %ptr_unsigned = getelementptr i32, i32* %p, i32 4095 213 %val_unsigned = load atomic i32, i32* %ptr_unsigned monotonic, align 4 214 ; CHECK: ldr {{w[0-9]+}}, [x0, #16380] 215 216 %ptr_regoff = getelementptr i32, i32* %p, i32 %off32 217 %val_regoff = load atomic i32, i32* %ptr_regoff unordered, align 4 218 %tot1 = add i32 %val_unsigned, %val_regoff 219 ; CHECK: ldr {{w[0-9]+}}, [x0, w1, sxtw #2] 220 221 %ptr_unscaled = getelementptr i32, i32* %p, i32 -64 222 %val_unscaled = load atomic i32, i32* %ptr_unscaled monotonic, align 4 223 %tot2 = add i32 %tot1, %val_unscaled 224 ; CHECK: ldur {{w[0-9]+}}, [x0, #-256] 225 226 %ptr_random = getelementptr i32, i32* %p, i32 297984 ; 0x123000/4 (i.e. ADD imm) 227 %val_random = load atomic i32, i32* %ptr_random unordered, align 4 228 %tot3 = add i32 %tot2, %val_random 229 ; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 230 ; CHECK: ldr {{w[0-9]+}}, [x[[ADDR]]] 231 232 ret i32 %tot3 233 } 234 235 define i64 @atomic_load_relaxed_64(i64* %p, i32 %off32) #0 { 236 ; CHECK-LABEL: atomic_load_relaxed_64: 237 %ptr_unsigned = getelementptr i64, i64* %p, i32 4095 238 %val_unsigned = load atomic i64, i64* %ptr_unsigned monotonic, align 8 239 ; CHECK: ldr {{x[0-9]+}}, [x0, #32760] 240 241 %ptr_regoff = getelementptr i64, i64* %p, i32 %off32 242 %val_regoff = load atomic i64, i64* %ptr_regoff unordered, align 8 243 %tot1 = add i64 %val_unsigned, %val_regoff 244 ; CHECK: ldr {{x[0-9]+}}, [x0, w1, sxtw #3] 245 246 %ptr_unscaled = getelementptr i64, i64* %p, i32 -32 247 %val_unscaled = load atomic i64, i64* %ptr_unscaled monotonic, align 8 248 %tot2 = add i64 %tot1, %val_unscaled 249 ; CHECK: ldur {{x[0-9]+}}, [x0, #-256] 250 251 %ptr_random = getelementptr i64, i64* %p, i32 148992 ; 0x123000/8 (i.e. ADD imm) 252 %val_random = load atomic i64, i64* %ptr_random unordered, align 8 253 %tot3 = add i64 %tot2, %val_random 254 ; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 255 ; CHECK: ldr {{x[0-9]+}}, [x[[ADDR]]] 256 257 ret i64 %tot3 258 } 259 260 261 define void @atomc_store(i32* %p) #0 { 262 store atomic i32 4, i32* %p seq_cst, align 4 263 ret void 264 ; CHECK-LABEL: atomc_store: 265 ; CHECK: stlr 266 } 267 268 define void @atomic_store_relaxed_8(i8* %p, i32 %off32, i8 %val) #0 { 269 ; CHECK-LABEL: atomic_store_relaxed_8: 270 %ptr_unsigned = getelementptr i8, i8* %p, i32 4095 271 store atomic i8 %val, i8* %ptr_unsigned monotonic, align 1 272 ; CHECK: strb {{w[0-9]+}}, [x0, #4095] 273 274 %ptr_regoff = getelementptr i8, i8* %p, i32 %off32 275 store atomic i8 %val, i8* %ptr_regoff unordered, align 1 276 ; CHECK: strb {{w[0-9]+}}, [x0, w1, sxtw] 277 278 %ptr_unscaled = getelementptr i8, i8* %p, i32 -256 279 store atomic i8 %val, i8* %ptr_unscaled monotonic, align 1 280 ; CHECK: sturb {{w[0-9]+}}, [x0, #-256] 281 282 %ptr_random = getelementptr i8, i8* %p, i32 1191936 ; 0x123000 (i.e. ADD imm) 283 store atomic i8 %val, i8* %ptr_random unordered, align 1 284 ; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 285 ; CHECK: strb {{w[0-9]+}}, [x[[ADDR]]] 286 287 ret void 288 } 289 290 define void @atomic_store_relaxed_16(i16* %p, i32 %off32, i16 %val) #0 { 291 ; CHECK-LABEL: atomic_store_relaxed_16: 292 %ptr_unsigned = getelementptr i16, i16* %p, i32 4095 293 store atomic i16 %val, i16* %ptr_unsigned monotonic, align 2 294 ; CHECK: strh {{w[0-9]+}}, [x0, #8190] 295 296 %ptr_regoff = getelementptr i16, i16* %p, i32 %off32 297 store atomic i16 %val, i16* %ptr_regoff unordered, align 2 298 ; CHECK: strh {{w[0-9]+}}, [x0, w1, sxtw #1] 299 300 %ptr_unscaled = getelementptr i16, i16* %p, i32 -128 301 store atomic i16 %val, i16* %ptr_unscaled monotonic, align 2 302 ; CHECK: sturh {{w[0-9]+}}, [x0, #-256] 303 304 %ptr_random = getelementptr i16, i16* %p, i32 595968 ; 0x123000/2 (i.e. ADD imm) 305 store atomic i16 %val, i16* %ptr_random unordered, align 2 306 ; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 307 ; CHECK: strh {{w[0-9]+}}, [x[[ADDR]]] 308 309 ret void 310 } 311 312 define void @atomic_store_relaxed_32(i32* %p, i32 %off32, i32 %val) #0 { 313 ; CHECK-LABEL: atomic_store_relaxed_32: 314 %ptr_unsigned = getelementptr i32, i32* %p, i32 4095 315 store atomic i32 %val, i32* %ptr_unsigned monotonic, align 4 316 ; CHECK: str {{w[0-9]+}}, [x0, #16380] 317 318 %ptr_regoff = getelementptr i32, i32* %p, i32 %off32 319 store atomic i32 %val, i32* %ptr_regoff unordered, align 4 320 ; CHECK: str {{w[0-9]+}}, [x0, w1, sxtw #2] 321 322 %ptr_unscaled = getelementptr i32, i32* %p, i32 -64 323 store atomic i32 %val, i32* %ptr_unscaled monotonic, align 4 324 ; CHECK: stur {{w[0-9]+}}, [x0, #-256] 325 326 %ptr_random = getelementptr i32, i32* %p, i32 297984 ; 0x123000/4 (i.e. ADD imm) 327 store atomic i32 %val, i32* %ptr_random unordered, align 4 328 ; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 329 ; CHECK: str {{w[0-9]+}}, [x[[ADDR]]] 330 331 ret void 332 } 333 334 define void @atomic_store_relaxed_64(i64* %p, i32 %off32, i64 %val) #0 { 335 ; CHECK-LABEL: atomic_store_relaxed_64: 336 %ptr_unsigned = getelementptr i64, i64* %p, i32 4095 337 store atomic i64 %val, i64* %ptr_unsigned monotonic, align 8 338 ; CHECK: str {{x[0-9]+}}, [x0, #32760] 339 340 %ptr_regoff = getelementptr i64, i64* %p, i32 %off32 341 store atomic i64 %val, i64* %ptr_regoff unordered, align 8 342 ; CHECK: str {{x[0-9]+}}, [x0, w1, sxtw #3] 343 344 %ptr_unscaled = getelementptr i64, i64* %p, i32 -32 345 store atomic i64 %val, i64* %ptr_unscaled monotonic, align 8 346 ; CHECK: stur {{x[0-9]+}}, [x0, #-256] 347 348 %ptr_random = getelementptr i64, i64* %p, i32 148992 ; 0x123000/8 (i.e. ADD imm) 349 store atomic i64 %val, i64* %ptr_random unordered, align 8 350 ; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 351 ; CHECK: str {{x[0-9]+}}, [x[[ADDR]]] 352 353 ret void 354 } 355 356 ; rdar://11531169 357 ; rdar://11531308 358 359 %"class.X::Atomic" = type { %struct.x_atomic_t } 360 %struct.x_atomic_t = type { i32 } 361 362 @counter = external hidden global %"class.X::Atomic", align 4 363 364 define i32 @next_id() nounwind optsize ssp align 2 { 365 entry: 366 %0 = atomicrmw add i32* getelementptr inbounds (%"class.X::Atomic", %"class.X::Atomic"* @counter, i64 0, i32 0, i32 0), i32 1 seq_cst 367 %add.i = add i32 %0, 1 368 %tobool = icmp eq i32 %add.i, 0 369 br i1 %tobool, label %if.else, label %return 370 371 if.else: ; preds = %entry 372 %1 = atomicrmw add i32* getelementptr inbounds (%"class.X::Atomic", %"class.X::Atomic"* @counter, i64 0, i32 0, i32 0), i32 1 seq_cst 373 %add.i2 = add i32 %1, 1 374 br label %return 375 376 return: ; preds = %if.else, %entry 377 %retval.0 = phi i32 [ %add.i2, %if.else ], [ %add.i, %entry ] 378 ret i32 %retval.0 379 } 380 381 attributes #0 = { nounwind } 382
