1 ; RUN: llc < %s -march=arm64 -asm-verbose=false -verify-machineinstrs -mcpu=cyclone | FileCheck %s 2 3 define i32 @val_compare_and_swap(i32* %p, i32 %cmp, i32 %new) #0 { 4 ; CHECK-LABEL: val_compare_and_swap: 5 ; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x0 6 ; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: 7 ; CHECK-NEXT: ldaxr [[RESULT:w[0-9]+]], [x[[ADDR]]] 8 ; CHECK-NEXT: cmp [[RESULT]], w1 9 ; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]] 10 ; CHECK-NEXT: stxr [[SCRATCH_REG:w[0-9]+]], w2, [x[[ADDR]]] 11 ; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]] 12 ; CHECK-NEXT: b [[EXITBB:.?LBB[0-9_]+]] 13 ; CHECK-NEXT: [[FAILBB]]: 14 ; CHECK-NEXT: clrex 15 ; CHECK-NEXT: [[EXITBB]]: 16 %pair = cmpxchg i32* %p, i32 %cmp, i32 %new acquire acquire 17 %val = extractvalue { i32, i1 } %pair, 0 18 ret i32 %val 19 } 20 21 define i32 @val_compare_and_swap_from_load(i32* %p, i32 %cmp, i32* %pnew) #0 { 22 ; CHECK-LABEL: val_compare_and_swap_from_load: 23 ; CHECK-NEXT: ldr [[NEW:w[0-9]+]], [x2] 24 ; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: 25 ; CHECK-NEXT: ldaxr [[RESULT:w[0-9]+]], [x0] 26 ; CHECK-NEXT: cmp [[RESULT]], w1 27 ; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]] 28 ; CHECK-NEXT: stxr [[SCRATCH_REG:w[0-9]+]], [[NEW]], [x0] 29 ; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]] 30 ; CHECK-NEXT: b [[EXITBB:.?LBB[0-9_]+]] 31 ; CHECK-NEXT: [[FAILBB]]: 32 ; CHECK-NEXT: clrex 33 ; CHECK-NEXT: [[EXITBB]]: 34 %new = load i32, i32* %pnew 35 %pair = cmpxchg i32* %p, i32 %cmp, i32 %new acquire acquire 36 %val = extractvalue { i32, i1 } %pair, 0 37 ret i32 %val 38 } 39 40 define i32 @val_compare_and_swap_rel(i32* %p, i32 %cmp, i32 %new) #0 { 41 ; CHECK-LABEL: val_compare_and_swap_rel: 42 ; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x0 43 ; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: 44 ; CHECK-NEXT: ldaxr [[RESULT:w[0-9]+]], [x[[ADDR]] 45 ; CHECK-NEXT: cmp [[RESULT]], w1 46 ; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]] 47 ; CHECK-NEXT: stlxr [[SCRATCH_REG:w[0-9]+]], w2, [x[[ADDR]] 48 ; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]] 49 ; CHECK-NEXT: b [[EXITBB:.?LBB[0-9_]+]] 50 ; CHECK-NEXT: [[FAILBB]]: 51 ; CHECK-NEXT: clrex 52 ; CHECK-NEXT: [[EXITBB]]: 53 %pair = cmpxchg i32* %p, i32 %cmp, i32 %new acq_rel monotonic 54 %val = extractvalue { i32, i1 } %pair, 0 55 ret i32 %val 56 } 57 58 define i64 @val_compare_and_swap_64(i64* %p, i64 %cmp, i64 %new) #0 { 59 ; CHECK-LABEL: val_compare_and_swap_64: 60 ; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x0 61 ; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: 62 ; CHECK-NEXT: ldxr [[RESULT:x[0-9]+]], [x[[ADDR]]] 63 ; CHECK-NEXT: cmp [[RESULT]], x1 64 ; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]] 65 ; CHECK-NEXT: stxr [[SCRATCH_REG:w[0-9]+]], x2, [x[[ADDR]]] 66 ; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]] 67 ; CHECK-NEXT: b [[EXITBB:.?LBB[0-9_]+]] 68 ; CHECK-NEXT: [[FAILBB]]: 69 ; CHECK-NEXT: clrex 70 ; CHECK-NEXT: [[EXITBB]]: 71 %pair = cmpxchg i64* %p, i64 %cmp, i64 %new monotonic monotonic 72 %val = extractvalue { i64, i1 } %pair, 0 73 ret i64 %val 74 } 75 76 define i32 @fetch_and_nand(i32* %p) #0 { 77 ; CHECK-LABEL: fetch_and_nand: 78 ; CHECK: [[TRYBB:.?LBB[0-9_]+]]: 79 ; CHECK: ldxr w[[DEST_REG:[0-9]+]], [x0] 80 ; CHECK: mvn [[TMP_REG:w[0-9]+]], w[[DEST_REG]] 81 ; CHECK: orr [[SCRATCH2_REG:w[0-9]+]], [[TMP_REG]], #0xfffffff8 82 ; CHECK-NOT: stlxr [[SCRATCH2_REG]], [[SCRATCH2_REG]] 83 ; CHECK: stlxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x0] 84 ; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]] 85 ; CHECK: mov x0, x[[DEST_REG]] 86 %val = atomicrmw nand i32* %p, i32 7 release 87 ret i32 %val 88 } 89 90 define i64 @fetch_and_nand_64(i64* %p) #0 { 91 ; CHECK-LABEL: fetch_and_nand_64: 92 ; CHECK: mov x[[ADDR:[0-9]+]], x0 93 ; CHECK: [[TRYBB:.?LBB[0-9_]+]]: 94 ; CHECK: ldaxr x[[DEST_REG:[0-9]+]], [x[[ADDR]]] 95 ; CHECK: mvn w[[TMP_REG:[0-9]+]], w[[DEST_REG]] 96 ; CHECK: orr [[SCRATCH2_REG:x[0-9]+]], x[[TMP_REG]], #0xfffffffffffffff8 97 ; CHECK: stlxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x[[ADDR]]] 98 ; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]] 99 100 %val = atomicrmw nand i64* %p, i64 7 acq_rel 101 ret i64 %val 102 } 103 104 define i32 @fetch_and_or(i32* %p) #0 { 105 ; CHECK-LABEL: fetch_and_or: 106 ; CHECK: movz [[OLDVAL_REG:w[0-9]+]], #0x5 107 ; CHECK: [[TRYBB:.?LBB[0-9_]+]]: 108 ; CHECK: ldaxr w[[DEST_REG:[0-9]+]], [x0] 109 ; CHECK: orr [[SCRATCH2_REG:w[0-9]+]], w[[DEST_REG]], [[OLDVAL_REG]] 110 ; CHECK-NOT: stlxr [[SCRATCH2_REG]], [[SCRATCH2_REG]] 111 ; CHECK: stlxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x0] 112 ; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]] 113 ; CHECK: mov x0, x[[DEST_REG]] 114 %val = atomicrmw or i32* %p, i32 5 seq_cst 115 ret i32 %val 116 } 117 118 define i64 @fetch_and_or_64(i64* %p) #0 { 119 ; CHECK: fetch_and_or_64: 120 ; CHECK: mov x[[ADDR:[0-9]+]], x0 121 ; CHECK: [[TRYBB:.?LBB[0-9_]+]]: 122 ; CHECK: ldxr [[DEST_REG:x[0-9]+]], [x[[ADDR]]] 123 ; CHECK: orr [[SCRATCH2_REG:x[0-9]+]], [[DEST_REG]], #0x7 124 ; CHECK: stxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x[[ADDR]]] 125 ; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]] 126 %val = atomicrmw or i64* %p, i64 7 monotonic 127 ret i64 %val 128 } 129 130 define void @acquire_fence() #0 { 131 fence acquire 132 ret void 133 ; CHECK-LABEL: acquire_fence: 134 ; CHECK: dmb ishld 135 } 136 137 define void @release_fence() #0 { 138 fence release 139 ret void 140 ; CHECK-LABEL: release_fence: 141 ; CHECK: dmb ish{{$}} 142 } 143 144 define void @seq_cst_fence() #0 { 145 fence seq_cst 146 ret void 147 ; CHECK-LABEL: seq_cst_fence: 148 ; CHECK: dmb ish{{$}} 149 } 150 151 define i32 @atomic_load(i32* %p) #0 { 152 %r = load atomic i32, i32* %p seq_cst, align 4 153 ret i32 %r 154 ; CHECK-LABEL: atomic_load: 155 ; CHECK: ldar 156 } 157 158 define i8 @atomic_load_relaxed_8(i8* %p, i32 %off32) #0 { 159 ; CHECK-LABEL: atomic_load_relaxed_8: 160 %ptr_unsigned = getelementptr i8, i8* %p, i32 4095 161 %val_unsigned = load atomic i8, i8* %ptr_unsigned monotonic, align 1 162 ; CHECK: ldrb {{w[0-9]+}}, [x0, #4095] 163 164 %ptr_regoff = getelementptr i8, i8* %p, i32 %off32 165 %val_regoff = load atomic i8, i8* %ptr_regoff unordered, align 1 166 %tot1 = add i8 %val_unsigned, %val_regoff 167 ; CHECK: ldrb {{w[0-9]+}}, [x0, w1, sxtw] 168 169 %ptr_unscaled = getelementptr i8, i8* %p, i32 -256 170 %val_unscaled = load atomic i8, i8* %ptr_unscaled monotonic, align 1 171 %tot2 = add i8 %tot1, %val_unscaled 172 ; CHECK: ldurb {{w[0-9]+}}, [x0, #-256] 173 174 %ptr_random = getelementptr i8, i8* %p, i32 1191936 ; 0x123000 (i.e. ADD imm) 175 %val_random = load atomic i8, i8* %ptr_random unordered, align 1 176 %tot3 = add i8 %tot2, %val_random 177 ; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 178 ; CHECK: ldrb {{w[0-9]+}}, [x[[ADDR]]] 179 180 ret i8 %tot3 181 } 182 183 define i16 @atomic_load_relaxed_16(i16* %p, i32 %off32) #0 { 184 ; CHECK-LABEL: atomic_load_relaxed_16: 185 %ptr_unsigned = getelementptr i16, i16* %p, i32 4095 186 %val_unsigned = load atomic i16, i16* %ptr_unsigned monotonic, align 2 187 ; CHECK: ldrh {{w[0-9]+}}, [x0, #8190] 188 189 %ptr_regoff = getelementptr i16, i16* %p, i32 %off32 190 %val_regoff = load atomic i16, i16* %ptr_regoff unordered, align 2 191 %tot1 = add i16 %val_unsigned, %val_regoff 192 ; CHECK: ldrh {{w[0-9]+}}, [x0, w1, sxtw #1] 193 194 %ptr_unscaled = getelementptr i16, i16* %p, i32 -128 195 %val_unscaled = load atomic i16, i16* %ptr_unscaled monotonic, align 2 196 %tot2 = add i16 %tot1, %val_unscaled 197 ; CHECK: ldurh {{w[0-9]+}}, [x0, #-256] 198 199 %ptr_random = getelementptr i16, i16* %p, i32 595968 ; 0x123000/2 (i.e. ADD imm) 200 %val_random = load atomic i16, i16* %ptr_random unordered, align 2 201 %tot3 = add i16 %tot2, %val_random 202 ; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 203 ; CHECK: ldrh {{w[0-9]+}}, [x[[ADDR]]] 204 205 ret i16 %tot3 206 } 207 208 define i32 @atomic_load_relaxed_32(i32* %p, i32 %off32) #0 { 209 ; CHECK-LABEL: atomic_load_relaxed_32: 210 %ptr_unsigned = getelementptr i32, i32* %p, i32 4095 211 %val_unsigned = load atomic i32, i32* %ptr_unsigned monotonic, align 4 212 ; CHECK: ldr {{w[0-9]+}}, [x0, #16380] 213 214 %ptr_regoff = getelementptr i32, i32* %p, i32 %off32 215 %val_regoff = load atomic i32, i32* %ptr_regoff unordered, align 4 216 %tot1 = add i32 %val_unsigned, %val_regoff 217 ; CHECK: ldr {{w[0-9]+}}, [x0, w1, sxtw #2] 218 219 %ptr_unscaled = getelementptr i32, i32* %p, i32 -64 220 %val_unscaled = load atomic i32, i32* %ptr_unscaled monotonic, align 4 221 %tot2 = add i32 %tot1, %val_unscaled 222 ; CHECK: ldur {{w[0-9]+}}, [x0, #-256] 223 224 %ptr_random = getelementptr i32, i32* %p, i32 297984 ; 0x123000/4 (i.e. ADD imm) 225 %val_random = load atomic i32, i32* %ptr_random unordered, align 4 226 %tot3 = add i32 %tot2, %val_random 227 ; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 228 ; CHECK: ldr {{w[0-9]+}}, [x[[ADDR]]] 229 230 ret i32 %tot3 231 } 232 233 define i64 @atomic_load_relaxed_64(i64* %p, i32 %off32) #0 { 234 ; CHECK-LABEL: atomic_load_relaxed_64: 235 %ptr_unsigned = getelementptr i64, i64* %p, i32 4095 236 %val_unsigned = load atomic i64, i64* %ptr_unsigned monotonic, align 8 237 ; CHECK: ldr {{x[0-9]+}}, [x0, #32760] 238 239 %ptr_regoff = getelementptr i64, i64* %p, i32 %off32 240 %val_regoff = load atomic i64, i64* %ptr_regoff unordered, align 8 241 %tot1 = add i64 %val_unsigned, %val_regoff 242 ; CHECK: ldr {{x[0-9]+}}, [x0, w1, sxtw #3] 243 244 %ptr_unscaled = getelementptr i64, i64* %p, i32 -32 245 %val_unscaled = load atomic i64, i64* %ptr_unscaled monotonic, align 8 246 %tot2 = add i64 %tot1, %val_unscaled 247 ; CHECK: ldur {{x[0-9]+}}, [x0, #-256] 248 249 %ptr_random = getelementptr i64, i64* %p, i32 148992 ; 0x123000/8 (i.e. ADD imm) 250 %val_random = load atomic i64, i64* %ptr_random unordered, align 8 251 %tot3 = add i64 %tot2, %val_random 252 ; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 253 ; CHECK: ldr {{x[0-9]+}}, [x[[ADDR]]] 254 255 ret i64 %tot3 256 } 257 258 259 define void @atomc_store(i32* %p) #0 { 260 store atomic i32 4, i32* %p seq_cst, align 4 261 ret void 262 ; CHECK-LABEL: atomc_store: 263 ; CHECK: stlr 264 } 265 266 define void @atomic_store_relaxed_8(i8* %p, i32 %off32, i8 %val) #0 { 267 ; CHECK-LABEL: atomic_store_relaxed_8: 268 %ptr_unsigned = getelementptr i8, i8* %p, i32 4095 269 store atomic i8 %val, i8* %ptr_unsigned monotonic, align 1 270 ; CHECK: strb {{w[0-9]+}}, [x0, #4095] 271 272 %ptr_regoff = getelementptr i8, i8* %p, i32 %off32 273 store atomic i8 %val, i8* %ptr_regoff unordered, align 1 274 ; CHECK: strb {{w[0-9]+}}, [x0, w1, sxtw] 275 276 %ptr_unscaled = getelementptr i8, i8* %p, i32 -256 277 store atomic i8 %val, i8* %ptr_unscaled monotonic, align 1 278 ; CHECK: sturb {{w[0-9]+}}, [x0, #-256] 279 280 %ptr_random = getelementptr i8, i8* %p, i32 1191936 ; 0x123000 (i.e. ADD imm) 281 store atomic i8 %val, i8* %ptr_random unordered, align 1 282 ; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 283 ; CHECK: strb {{w[0-9]+}}, [x[[ADDR]]] 284 285 ret void 286 } 287 288 define void @atomic_store_relaxed_16(i16* %p, i32 %off32, i16 %val) #0 { 289 ; CHECK-LABEL: atomic_store_relaxed_16: 290 %ptr_unsigned = getelementptr i16, i16* %p, i32 4095 291 store atomic i16 %val, i16* %ptr_unsigned monotonic, align 2 292 ; CHECK: strh {{w[0-9]+}}, [x0, #8190] 293 294 %ptr_regoff = getelementptr i16, i16* %p, i32 %off32 295 store atomic i16 %val, i16* %ptr_regoff unordered, align 2 296 ; CHECK: strh {{w[0-9]+}}, [x0, w1, sxtw #1] 297 298 %ptr_unscaled = getelementptr i16, i16* %p, i32 -128 299 store atomic i16 %val, i16* %ptr_unscaled monotonic, align 2 300 ; CHECK: sturh {{w[0-9]+}}, [x0, #-256] 301 302 %ptr_random = getelementptr i16, i16* %p, i32 595968 ; 0x123000/2 (i.e. ADD imm) 303 store atomic i16 %val, i16* %ptr_random unordered, align 2 304 ; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 305 ; CHECK: strh {{w[0-9]+}}, [x[[ADDR]]] 306 307 ret void 308 } 309 310 define void @atomic_store_relaxed_32(i32* %p, i32 %off32, i32 %val) #0 { 311 ; CHECK-LABEL: atomic_store_relaxed_32: 312 %ptr_unsigned = getelementptr i32, i32* %p, i32 4095 313 store atomic i32 %val, i32* %ptr_unsigned monotonic, align 4 314 ; CHECK: str {{w[0-9]+}}, [x0, #16380] 315 316 %ptr_regoff = getelementptr i32, i32* %p, i32 %off32 317 store atomic i32 %val, i32* %ptr_regoff unordered, align 4 318 ; CHECK: str {{w[0-9]+}}, [x0, w1, sxtw #2] 319 320 %ptr_unscaled = getelementptr i32, i32* %p, i32 -64 321 store atomic i32 %val, i32* %ptr_unscaled monotonic, align 4 322 ; CHECK: stur {{w[0-9]+}}, [x0, #-256] 323 324 %ptr_random = getelementptr i32, i32* %p, i32 297984 ; 0x123000/4 (i.e. ADD imm) 325 store atomic i32 %val, i32* %ptr_random unordered, align 4 326 ; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 327 ; CHECK: str {{w[0-9]+}}, [x[[ADDR]]] 328 329 ret void 330 } 331 332 define void @atomic_store_relaxed_64(i64* %p, i32 %off32, i64 %val) #0 { 333 ; CHECK-LABEL: atomic_store_relaxed_64: 334 %ptr_unsigned = getelementptr i64, i64* %p, i32 4095 335 store atomic i64 %val, i64* %ptr_unsigned monotonic, align 8 336 ; CHECK: str {{x[0-9]+}}, [x0, #32760] 337 338 %ptr_regoff = getelementptr i64, i64* %p, i32 %off32 339 store atomic i64 %val, i64* %ptr_regoff unordered, align 8 340 ; CHECK: str {{x[0-9]+}}, [x0, w1, sxtw #3] 341 342 %ptr_unscaled = getelementptr i64, i64* %p, i32 -32 343 store atomic i64 %val, i64* %ptr_unscaled monotonic, align 8 344 ; CHECK: stur {{x[0-9]+}}, [x0, #-256] 345 346 %ptr_random = getelementptr i64, i64* %p, i32 148992 ; 0x123000/8 (i.e. ADD imm) 347 store atomic i64 %val, i64* %ptr_random unordered, align 8 348 ; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 349 ; CHECK: str {{x[0-9]+}}, [x[[ADDR]]] 350 351 ret void 352 } 353 354 ; rdar://11531169 355 ; rdar://11531308 356 357 %"class.X::Atomic" = type { %struct.x_atomic_t } 358 %struct.x_atomic_t = type { i32 } 359 360 @counter = external hidden global %"class.X::Atomic", align 4 361 362 define i32 @next_id() nounwind optsize ssp align 2 { 363 entry: 364 %0 = atomicrmw add i32* getelementptr inbounds (%"class.X::Atomic", %"class.X::Atomic"* @counter, i64 0, i32 0, i32 0), i32 1 seq_cst 365 %add.i = add i32 %0, 1 366 %tobool = icmp eq i32 %add.i, 0 367 br i1 %tobool, label %if.else, label %return 368 369 if.else: ; preds = %entry 370 %1 = atomicrmw add i32* getelementptr inbounds (%"class.X::Atomic", %"class.X::Atomic"* @counter, i64 0, i32 0, i32 0), i32 1 seq_cst 371 %add.i2 = add i32 %1, 1 372 br label %return 373 374 return: ; preds = %if.else, %entry 375 %retval.0 = phi i32 [ %add.i2, %if.else ], [ %add.i, %entry ] 376 ret i32 %retval.0 377 } 378 379 attributes #0 = { nounwind } 380
