1 ; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK 2 ; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-REG 3 4 5 ; Point of CHECK-REG is to make sure UNPREDICTABLE instructions aren't created 6 ; (i.e. reusing a register for status & data in store exclusive). 7 ; CHECK-REG-NOT: stlxrb w[[NEW:[0-9]+]], w[[NEW]], [x{{[0-9]+}}] 8 ; CHECK-REG-NOT: stlxrb w[[NEW:[0-9]+]], x[[NEW]], [x{{[0-9]+}}] 9 10 @var8 = global i8 0 11 @var16 = global i16 0 12 @var32 = global i32 0 13 @var64 = global i64 0 14 15 define i8 @test_atomic_load_add_i8(i8 %offset) nounwind { 16 ; CHECK-LABEL: test_atomic_load_add_i8: 17 %old = atomicrmw add i8* @var8, i8 %offset seq_cst 18 ; CHECK-NOT: dmb 19 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 20 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 21 22 ; CHECK: .LBB{{[0-9]+}}_1: 23 ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]] 24 ; w0 below is a reasonable guess but could change: it certainly comes into the 25 ; function there. 26 ; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0 27 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 28 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 29 ; CHECK-NOT: dmb 30 31 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 32 ret i8 %old 33 } 34 35 define i16 @test_atomic_load_add_i16(i16 %offset) nounwind { 36 ; CHECK-LABEL: test_atomic_load_add_i16: 37 %old = atomicrmw add i16* @var16, i16 %offset acquire 38 ; CHECK-NOT: dmb 39 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 40 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 41 42 ; CHECK: .LBB{{[0-9]+}}_1: 43 ; ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]] 44 ; w0 below is a reasonable guess but could change: it certainly comes into the 45 ; function there. 46 ; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0 47 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 48 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 49 ; CHECK-NOT: dmb 50 51 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 52 ret i16 %old 53 } 54 55 define i32 @test_atomic_load_add_i32(i32 %offset) nounwind { 56 ; CHECK-LABEL: test_atomic_load_add_i32: 57 %old = atomicrmw add i32* @var32, i32 %offset release 58 ; CHECK-NOT: dmb 59 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 60 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 61 62 ; CHECK: .LBB{{[0-9]+}}_1: 63 ; ; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]] 64 ; w0 below is a reasonable guess but could change: it certainly comes into the 65 ; function there. 66 ; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0 67 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 68 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 69 ; CHECK-NOT: dmb 70 71 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 72 ret i32 %old 73 } 74 75 define i64 @test_atomic_load_add_i64(i64 %offset) nounwind { 76 ; CHECK-LABEL: test_atomic_load_add_i64: 77 %old = atomicrmw add i64* @var64, i64 %offset monotonic 78 ; CHECK-NOT: dmb 79 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 80 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 81 82 ; CHECK: .LBB{{[0-9]+}}_1: 83 ; ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]] 84 ; x0 below is a reasonable guess but could change: it certainly comes into the 85 ; function there. 86 ; CHECK-NEXT: add [[NEW:x[0-9]+]], x[[OLD]], x0 87 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 88 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 89 ; CHECK-NOT: dmb 90 91 ; CHECK: mov x0, x[[OLD]] 92 ret i64 %old 93 } 94 95 define i8 @test_atomic_load_sub_i8(i8 %offset) nounwind { 96 ; CHECK-LABEL: test_atomic_load_sub_i8: 97 %old = atomicrmw sub i8* @var8, i8 %offset monotonic 98 ; CHECK-NOT: dmb 99 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 100 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 101 102 ; CHECK: .LBB{{[0-9]+}}_1: 103 ; ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]] 104 ; w0 below is a reasonable guess but could change: it certainly comes into the 105 ; function there. 106 ; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0 107 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 108 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 109 ; CHECK-NOT: dmb 110 111 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 112 ret i8 %old 113 } 114 115 define i16 @test_atomic_load_sub_i16(i16 %offset) nounwind { 116 ; CHECK-LABEL: test_atomic_load_sub_i16: 117 %old = atomicrmw sub i16* @var16, i16 %offset release 118 ; CHECK-NOT: dmb 119 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 120 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 121 122 ; CHECK: .LBB{{[0-9]+}}_1: 123 ; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]] 124 ; w0 below is a reasonable guess but could change: it certainly comes into the 125 ; function there. 126 ; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0 127 ; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 128 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 129 ; CHECK-NOT: dmb 130 131 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 132 ret i16 %old 133 } 134 135 define i32 @test_atomic_load_sub_i32(i32 %offset) nounwind { 136 ; CHECK-LABEL: test_atomic_load_sub_i32: 137 %old = atomicrmw sub i32* @var32, i32 %offset acquire 138 ; CHECK-NOT: dmb 139 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 140 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 141 142 ; CHECK: .LBB{{[0-9]+}}_1: 143 ; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]] 144 ; w0 below is a reasonable guess but could change: it certainly comes into the 145 ; function there. 146 ; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0 147 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 148 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 149 ; CHECK-NOT: dmb 150 151 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 152 ret i32 %old 153 } 154 155 define i64 @test_atomic_load_sub_i64(i64 %offset) nounwind { 156 ; CHECK-LABEL: test_atomic_load_sub_i64: 157 %old = atomicrmw sub i64* @var64, i64 %offset seq_cst 158 ; CHECK-NOT: dmb 159 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 160 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 161 162 ; CHECK: .LBB{{[0-9]+}}_1: 163 ; ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]] 164 ; x0 below is a reasonable guess but could change: it certainly comes into the 165 ; function there. 166 ; CHECK-NEXT: sub [[NEW:x[0-9]+]], x[[OLD]], x0 167 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 168 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 169 ; CHECK-NOT: dmb 170 171 ; CHECK: mov x0, x[[OLD]] 172 ret i64 %old 173 } 174 175 define i8 @test_atomic_load_and_i8(i8 %offset) nounwind { 176 ; CHECK-LABEL: test_atomic_load_and_i8: 177 %old = atomicrmw and i8* @var8, i8 %offset release 178 ; CHECK-NOT: dmb 179 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 180 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 181 182 ; CHECK: .LBB{{[0-9]+}}_1: 183 ; ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]] 184 ; w0 below is a reasonable guess but could change: it certainly comes into the 185 ; function there. 186 ; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0 187 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 188 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 189 ; CHECK-NOT: dmb 190 191 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 192 ret i8 %old 193 } 194 195 define i16 @test_atomic_load_and_i16(i16 %offset) nounwind { 196 ; CHECK-LABEL: test_atomic_load_and_i16: 197 %old = atomicrmw and i16* @var16, i16 %offset monotonic 198 ; CHECK-NOT: dmb 199 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 200 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 201 202 ; CHECK: .LBB{{[0-9]+}}_1: 203 ; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]] 204 ; w0 below is a reasonable guess but could change: it certainly comes into the 205 ; function there. 206 ; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0 207 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 208 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 209 ; CHECK-NOT: dmb 210 211 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 212 ret i16 %old 213 } 214 215 define i32 @test_atomic_load_and_i32(i32 %offset) nounwind { 216 ; CHECK-LABEL: test_atomic_load_and_i32: 217 %old = atomicrmw and i32* @var32, i32 %offset seq_cst 218 ; CHECK-NOT: dmb 219 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 220 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 221 222 ; CHECK: .LBB{{[0-9]+}}_1: 223 ; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]] 224 ; w0 below is a reasonable guess but could change: it certainly comes into the 225 ; function there. 226 ; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0 227 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 228 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 229 ; CHECK-NOT: dmb 230 231 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 232 ret i32 %old 233 } 234 235 define i64 @test_atomic_load_and_i64(i64 %offset) nounwind { 236 ; CHECK-LABEL: test_atomic_load_and_i64: 237 %old = atomicrmw and i64* @var64, i64 %offset acquire 238 ; CHECK-NOT: dmb 239 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 240 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 241 242 ; CHECK: .LBB{{[0-9]+}}_1: 243 ; ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]] 244 ; x0 below is a reasonable guess but could change: it certainly comes into the 245 ; function there. 246 ; CHECK-NEXT: and [[NEW:x[0-9]+]], x[[OLD]], x0 247 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 248 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 249 ; CHECK-NOT: dmb 250 251 ; CHECK: mov x0, x[[OLD]] 252 ret i64 %old 253 } 254 255 define i8 @test_atomic_load_or_i8(i8 %offset) nounwind { 256 ; CHECK-LABEL: test_atomic_load_or_i8: 257 %old = atomicrmw or i8* @var8, i8 %offset seq_cst 258 ; CHECK-NOT: dmb 259 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 260 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 261 262 ; CHECK: .LBB{{[0-9]+}}_1: 263 ; ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]] 264 ; w0 below is a reasonable guess but could change: it certainly comes into the 265 ; function there. 266 ; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0 267 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 268 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 269 ; CHECK-NOT: dmb 270 271 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 272 ret i8 %old 273 } 274 275 define i16 @test_atomic_load_or_i16(i16 %offset) nounwind { 276 ; CHECK-LABEL: test_atomic_load_or_i16: 277 %old = atomicrmw or i16* @var16, i16 %offset monotonic 278 ; CHECK-NOT: dmb 279 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 280 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 281 282 ; CHECK: .LBB{{[0-9]+}}_1: 283 ; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]] 284 ; w0 below is a reasonable guess but could change: it certainly comes into the 285 ; function there. 286 ; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0 287 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 288 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 289 ; CHECK-NOT: dmb 290 291 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 292 ret i16 %old 293 } 294 295 define i32 @test_atomic_load_or_i32(i32 %offset) nounwind { 296 ; CHECK-LABEL: test_atomic_load_or_i32: 297 %old = atomicrmw or i32* @var32, i32 %offset acquire 298 ; CHECK-NOT: dmb 299 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 300 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 301 302 ; CHECK: .LBB{{[0-9]+}}_1: 303 ; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]] 304 ; w0 below is a reasonable guess but could change: it certainly comes into the 305 ; function there. 306 ; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0 307 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 308 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 309 ; CHECK-NOT: dmb 310 311 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 312 ret i32 %old 313 } 314 315 define i64 @test_atomic_load_or_i64(i64 %offset) nounwind { 316 ; CHECK-LABEL: test_atomic_load_or_i64: 317 %old = atomicrmw or i64* @var64, i64 %offset release 318 ; CHECK-NOT: dmb 319 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 320 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 321 322 ; CHECK: .LBB{{[0-9]+}}_1: 323 ; ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]] 324 ; x0 below is a reasonable guess but could change: it certainly comes into the 325 ; function there. 326 ; CHECK-NEXT: orr [[NEW:x[0-9]+]], x[[OLD]], x0 327 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 328 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 329 ; CHECK-NOT: dmb 330 331 ; CHECK: mov x0, x[[OLD]] 332 ret i64 %old 333 } 334 335 define i8 @test_atomic_load_xor_i8(i8 %offset) nounwind { 336 ; CHECK-LABEL: test_atomic_load_xor_i8: 337 %old = atomicrmw xor i8* @var8, i8 %offset acquire 338 ; CHECK-NOT: dmb 339 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 340 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 341 342 ; CHECK: .LBB{{[0-9]+}}_1: 343 ; ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]] 344 ; w0 below is a reasonable guess but could change: it certainly comes into the 345 ; function there. 346 ; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0 347 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 348 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 349 ; CHECK-NOT: dmb 350 351 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 352 ret i8 %old 353 } 354 355 define i16 @test_atomic_load_xor_i16(i16 %offset) nounwind { 356 ; CHECK-LABEL: test_atomic_load_xor_i16: 357 %old = atomicrmw xor i16* @var16, i16 %offset release 358 ; CHECK-NOT: dmb 359 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 360 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 361 362 ; CHECK: .LBB{{[0-9]+}}_1: 363 ; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]] 364 ; w0 below is a reasonable guess but could change: it certainly comes into the 365 ; function there. 366 ; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0 367 ; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 368 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 369 ; CHECK-NOT: dmb 370 371 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 372 ret i16 %old 373 } 374 375 define i32 @test_atomic_load_xor_i32(i32 %offset) nounwind { 376 ; CHECK-LABEL: test_atomic_load_xor_i32: 377 %old = atomicrmw xor i32* @var32, i32 %offset seq_cst 378 ; CHECK-NOT: dmb 379 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 380 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 381 382 ; CHECK: .LBB{{[0-9]+}}_1: 383 ; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]] 384 ; w0 below is a reasonable guess but could change: it certainly comes into the 385 ; function there. 386 ; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0 387 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 388 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 389 ; CHECK-NOT: dmb 390 391 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 392 ret i32 %old 393 } 394 395 define i64 @test_atomic_load_xor_i64(i64 %offset) nounwind { 396 ; CHECK-LABEL: test_atomic_load_xor_i64: 397 %old = atomicrmw xor i64* @var64, i64 %offset monotonic 398 ; CHECK-NOT: dmb 399 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 400 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 401 402 ; CHECK: .LBB{{[0-9]+}}_1: 403 ; ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]] 404 ; x0 below is a reasonable guess but could change: it certainly comes into the 405 ; function there. 406 ; CHECK-NEXT: eor [[NEW:x[0-9]+]], x[[OLD]], x0 407 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 408 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 409 ; CHECK-NOT: dmb 410 411 ; CHECK: mov x0, x[[OLD]] 412 ret i64 %old 413 } 414 415 define i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind { 416 ; CHECK-LABEL: test_atomic_load_xchg_i8: 417 %old = atomicrmw xchg i8* @var8, i8 %offset monotonic 418 ; CHECK-NOT: dmb 419 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 420 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 421 422 ; CHECK: .LBB{{[0-9]+}}_1: 423 ; ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]] 424 ; w0 below is a reasonable guess but could change: it certainly comes into the 425 ; function there. 426 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], w0, [x[[ADDR]]] 427 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 428 ; CHECK-NOT: dmb 429 430 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 431 ret i8 %old 432 } 433 434 define i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind { 435 ; CHECK-LABEL: test_atomic_load_xchg_i16: 436 %old = atomicrmw xchg i16* @var16, i16 %offset seq_cst 437 ; CHECK-NOT: dmb 438 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 439 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 440 441 ; CHECK: .LBB{{[0-9]+}}_1: 442 ; ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]] 443 ; w0 below is a reasonable guess but could change: it certainly comes into the 444 ; function there. 445 ; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], w0, [x[[ADDR]]] 446 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 447 ; CHECK-NOT: dmb 448 449 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 450 ret i16 %old 451 } 452 453 define i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind { 454 ; CHECK-LABEL: test_atomic_load_xchg_i32: 455 %old = atomicrmw xchg i32* @var32, i32 %offset release 456 ; CHECK-NOT: dmb 457 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 458 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 459 460 ; CHECK: .LBB{{[0-9]+}}_1: 461 ; ; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]] 462 ; w0 below is a reasonable guess but could change: it certainly comes into the 463 ; function there. 464 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], w0, [x[[ADDR]]] 465 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 466 ; CHECK-NOT: dmb 467 468 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 469 ret i32 %old 470 } 471 472 define i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind { 473 ; CHECK-LABEL: test_atomic_load_xchg_i64: 474 %old = atomicrmw xchg i64* @var64, i64 %offset acquire 475 ; CHECK-NOT: dmb 476 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 477 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 478 479 ; CHECK: .LBB{{[0-9]+}}_1: 480 ; ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]] 481 ; x0 below is a reasonable guess but could change: it certainly comes into the 482 ; function there. 483 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], x0, [x[[ADDR]]] 484 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 485 ; CHECK-NOT: dmb 486 487 ; CHECK: mov x0, x[[OLD]] 488 ret i64 %old 489 } 490 491 492 define i8 @test_atomic_load_min_i8(i8 %offset) nounwind { 493 ; CHECK-LABEL: test_atomic_load_min_i8: 494 %old = atomicrmw min i8* @var8, i8 %offset acquire 495 ; CHECK-NOT: dmb 496 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 497 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 498 499 ; CHECK: .LBB{{[0-9]+}}_1: 500 ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]] 501 ; w0 below is a reasonable guess but could change: it certainly comes into the 502 ; function there. 503 504 ; CHECK-NEXT: sxtb w[[OLD_EXT:[0-9]+]], w[[OLD]] 505 ; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxtb 506 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, le 507 508 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 509 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 510 ; CHECK-NOT: dmb 511 512 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]] 513 ret i8 %old 514 } 515 516 define i16 @test_atomic_load_min_i16(i16 %offset) nounwind { 517 ; CHECK-LABEL: test_atomic_load_min_i16: 518 %old = atomicrmw min i16* @var16, i16 %offset release 519 ; CHECK-NOT: dmb 520 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 521 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 522 523 ; CHECK: .LBB{{[0-9]+}}_1: 524 ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]] 525 ; w0 below is a reasonable guess but could change: it certainly comes into the 526 ; function there. 527 528 ; CHECK-NEXT: sxth w[[OLD_EXT:[0-9]+]], w[[OLD]] 529 ; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxth 530 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, le 531 532 533 ; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 534 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 535 ; CHECK-NOT: dmb 536 537 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]] 538 ret i16 %old 539 } 540 541 define i32 @test_atomic_load_min_i32(i32 %offset) nounwind { 542 ; CHECK-LABEL: test_atomic_load_min_i32: 543 %old = atomicrmw min i32* @var32, i32 %offset monotonic 544 ; CHECK-NOT: dmb 545 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 546 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 547 548 ; CHECK: .LBB{{[0-9]+}}_1: 549 ; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]] 550 ; w0 below is a reasonable guess but could change: it certainly comes into the 551 ; function there. 552 553 ; CHECK-NEXT: cmp w[[OLD]], w0 554 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, le 555 556 557 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 558 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 559 ; CHECK-NOT: dmb 560 561 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 562 ret i32 %old 563 } 564 565 define i64 @test_atomic_load_min_i64(i64 %offset) nounwind { 566 ; CHECK-LABEL: test_atomic_load_min_i64: 567 %old = atomicrmw min i64* @var64, i64 %offset seq_cst 568 ; CHECK-NOT: dmb 569 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 570 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 571 572 ; CHECK: .LBB{{[0-9]+}}_1: 573 ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]] 574 ; x0 below is a reasonable guess but could change: it certainly comes into the 575 ; function there. 576 577 ; CHECK-NEXT: cmp x[[OLD]], x0 578 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, le 579 580 581 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 582 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 583 ; CHECK-NOT: dmb 584 585 ; CHECK: mov x0, x[[OLD]] 586 ret i64 %old 587 } 588 589 define i8 @test_atomic_load_max_i8(i8 %offset) nounwind { 590 ; CHECK-LABEL: test_atomic_load_max_i8: 591 %old = atomicrmw max i8* @var8, i8 %offset seq_cst 592 ; CHECK-NOT: dmb 593 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 594 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 595 596 ; CHECK: .LBB{{[0-9]+}}_1: 597 ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]] 598 ; w0 below is a reasonable guess but could change: it certainly comes into the 599 ; function there. 600 601 ; CHECK-NEXT: sxtb w[[OLD_EXT:[0-9]+]], w[[OLD]] 602 ; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxtb 603 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt 604 605 606 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 607 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 608 ; CHECK-NOT: dmb 609 610 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]] 611 ret i8 %old 612 } 613 614 define i16 @test_atomic_load_max_i16(i16 %offset) nounwind { 615 ; CHECK-LABEL: test_atomic_load_max_i16: 616 %old = atomicrmw max i16* @var16, i16 %offset acquire 617 ; CHECK-NOT: dmb 618 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 619 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 620 621 ; CHECK: .LBB{{[0-9]+}}_1: 622 ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]] 623 ; w0 below is a reasonable guess but could change: it certainly comes into the 624 ; function there. 625 626 ; CHECK-NEXT: sxth w[[OLD_EXT:[0-9]+]], w[[OLD]] 627 ; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxth 628 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt 629 630 631 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 632 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 633 ; CHECK-NOT: dmb 634 635 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]] 636 ret i16 %old 637 } 638 639 define i32 @test_atomic_load_max_i32(i32 %offset) nounwind { 640 ; CHECK-LABEL: test_atomic_load_max_i32: 641 %old = atomicrmw max i32* @var32, i32 %offset release 642 ; CHECK-NOT: dmb 643 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 644 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 645 646 ; CHECK: .LBB{{[0-9]+}}_1: 647 ; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]] 648 ; w0 below is a reasonable guess but could change: it certainly comes into the 649 ; function there. 650 651 ; CHECK-NEXT: cmp w[[OLD]], w0 652 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt 653 654 655 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 656 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 657 ; CHECK-NOT: dmb 658 659 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 660 ret i32 %old 661 } 662 663 define i64 @test_atomic_load_max_i64(i64 %offset) nounwind { 664 ; CHECK-LABEL: test_atomic_load_max_i64: 665 %old = atomicrmw max i64* @var64, i64 %offset monotonic 666 ; CHECK-NOT: dmb 667 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 668 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 669 670 ; CHECK: .LBB{{[0-9]+}}_1: 671 ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]] 672 ; x0 below is a reasonable guess but could change: it certainly comes into the 673 ; function there. 674 675 ; CHECK-NEXT: cmp x[[OLD]], x0 676 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, gt 677 678 679 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 680 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 681 ; CHECK-NOT: dmb 682 683 ; CHECK: mov x0, x[[OLD]] 684 ret i64 %old 685 } 686 687 define i8 @test_atomic_load_umin_i8(i8 %offset) nounwind { 688 ; CHECK-LABEL: test_atomic_load_umin_i8: 689 %old = atomicrmw umin i8* @var8, i8 %offset monotonic 690 ; CHECK-NOT: dmb 691 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 692 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 693 694 ; CHECK: .LBB{{[0-9]+}}_1: 695 ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]] 696 ; w0 below is a reasonable guess but could change: it certainly comes into the 697 ; function there. 698 699 ; CHECK-NEXT: cmp w[[OLD]], w0, uxtb 700 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, ls 701 702 703 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 704 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 705 ; CHECK-NOT: dmb 706 707 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 708 ret i8 %old 709 } 710 711 define i16 @test_atomic_load_umin_i16(i16 %offset) nounwind { 712 ; CHECK-LABEL: test_atomic_load_umin_i16: 713 %old = atomicrmw umin i16* @var16, i16 %offset acquire 714 ; CHECK-NOT: dmb 715 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 716 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 717 718 ; CHECK: .LBB{{[0-9]+}}_1: 719 ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]] 720 ; w0 below is a reasonable guess but could change: it certainly comes into the 721 ; function there. 722 723 ; CHECK-NEXT: cmp w[[OLD]], w0, uxth 724 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, ls 725 726 727 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 728 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 729 ; CHECK-NOT: dmb 730 731 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 732 ret i16 %old 733 } 734 735 define i32 @test_atomic_load_umin_i32(i32 %offset) nounwind { 736 ; CHECK-LABEL: test_atomic_load_umin_i32: 737 %old = atomicrmw umin i32* @var32, i32 %offset seq_cst 738 ; CHECK-NOT: dmb 739 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 740 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 741 742 ; CHECK: .LBB{{[0-9]+}}_1: 743 ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]] 744 ; w0 below is a reasonable guess but could change: it certainly comes into the 745 ; function there. 746 747 ; CHECK-NEXT: cmp w[[OLD]], w0 748 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, ls 749 750 751 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 752 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 753 ; CHECK-NOT: dmb 754 755 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 756 ret i32 %old 757 } 758 759 define i64 @test_atomic_load_umin_i64(i64 %offset) nounwind { 760 ; CHECK-LABEL: test_atomic_load_umin_i64: 761 %old = atomicrmw umin i64* @var64, i64 %offset acq_rel 762 ; CHECK-NOT: dmb 763 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 764 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 765 766 ; CHECK: .LBB{{[0-9]+}}_1: 767 ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]] 768 ; x0 below is a reasonable guess but could change: it certainly comes into the 769 ; function there. 770 771 ; CHECK-NEXT: cmp x[[OLD]], x0 772 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, ls 773 774 775 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 776 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 777 ; CHECK-NOT: dmb 778 779 ; CHECK: mov x0, x[[OLD]] 780 ret i64 %old 781 } 782 783 define i8 @test_atomic_load_umax_i8(i8 %offset) nounwind { 784 ; CHECK-LABEL: test_atomic_load_umax_i8: 785 %old = atomicrmw umax i8* @var8, i8 %offset acq_rel 786 ; CHECK-NOT: dmb 787 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 788 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 789 790 ; CHECK: .LBB{{[0-9]+}}_1: 791 ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]] 792 ; w0 below is a reasonable guess but could change: it certainly comes into the 793 ; function there. 794 795 ; CHECK-NEXT: cmp w[[OLD]], w0, uxtb 796 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi 797 798 799 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 800 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 801 ; CHECK-NOT: dmb 802 803 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 804 ret i8 %old 805 } 806 807 define i16 @test_atomic_load_umax_i16(i16 %offset) nounwind { 808 ; CHECK-LABEL: test_atomic_load_umax_i16: 809 %old = atomicrmw umax i16* @var16, i16 %offset monotonic 810 ; CHECK-NOT: dmb 811 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 812 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 813 814 ; CHECK: .LBB{{[0-9]+}}_1: 815 ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]] 816 ; w0 below is a reasonable guess but could change: it certainly comes into the 817 ; function there. 818 819 ; CHECK-NEXT: cmp w[[OLD]], w0, uxth 820 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi 821 822 823 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 824 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 825 ; CHECK-NOT: dmb 826 827 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 828 ret i16 %old 829 } 830 831 define i32 @test_atomic_load_umax_i32(i32 %offset) nounwind { 832 ; CHECK-LABEL: test_atomic_load_umax_i32: 833 %old = atomicrmw umax i32* @var32, i32 %offset seq_cst 834 ; CHECK-NOT: dmb 835 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 836 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 837 838 ; CHECK: .LBB{{[0-9]+}}_1: 839 ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]] 840 ; w0 below is a reasonable guess but could change: it certainly comes into the 841 ; function there. 842 843 ; CHECK-NEXT: cmp w[[OLD]], w0 844 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi 845 846 847 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 848 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 849 ; CHECK-NOT: dmb 850 851 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 852 ret i32 %old 853 } 854 855 define i64 @test_atomic_load_umax_i64(i64 %offset) nounwind { 856 ; CHECK-LABEL: test_atomic_load_umax_i64: 857 %old = atomicrmw umax i64* @var64, i64 %offset release 858 ; CHECK-NOT: dmb 859 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 860 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 861 862 ; CHECK: .LBB{{[0-9]+}}_1: 863 ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]] 864 ; x0 below is a reasonable guess but could change: it certainly comes into the 865 ; function there. 866 867 ; CHECK-NEXT: cmp x[[OLD]], x0 868 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, hi 869 870 871 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]] 872 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1 873 ; CHECK-NOT: dmb 874 875 ; CHECK: mov x0, x[[OLD]] 876 ret i64 %old 877 } 878 879 define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind { 880 ; CHECK-LABEL: test_atomic_cmpxchg_i8: 881 %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire 882 %old = extractvalue { i8, i1 } %pair, 0 883 884 ; CHECK-NOT: dmb 885 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 886 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 887 888 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]: 889 ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]] 890 ; w0 below is a reasonable guess but could change: it certainly comes into the 891 ; function there. 892 ; CHECK-NEXT: cmp w[[OLD]], w0 893 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]] 894 ; CHECK: stxrb [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]] 895 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]] 896 ; CHECK-NOT: dmb 897 898 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 899 ret i8 %old 900 } 901 902 define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind { 903 ; CHECK-LABEL: test_atomic_cmpxchg_i16: 904 %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst 905 %old = extractvalue { i16, i1 } %pair, 0 906 907 ; CHECK-NOT: dmb 908 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 909 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 910 911 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]: 912 ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]] 913 ; w0 below is a reasonable guess but could change: it certainly comes into the 914 ; function there. 915 ; CHECK-NEXT: cmp w[[OLD]], w0 916 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]] 917 ; CHECK: stlxrh [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]] 918 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]] 919 ; CHECK-NOT: dmb 920 921 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 922 ret i16 %old 923 } 924 925 define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind { 926 ; CHECK-LABEL: test_atomic_cmpxchg_i32: 927 %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new release monotonic 928 %old = extractvalue { i32, i1 } %pair, 0 929 930 ; CHECK-NOT: dmb 931 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 932 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 933 934 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]: 935 ; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]] 936 ; w0 below is a reasonable guess but could change: it certainly comes into the 937 ; function there. 938 ; CHECK-NEXT: cmp w[[OLD]], w0 939 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]] 940 ; CHECK: stlxr [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]] 941 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]] 942 ; CHECK-NOT: dmb 943 944 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] 945 ret i32 %old 946 } 947 948 define void @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind { 949 ; CHECK-LABEL: test_atomic_cmpxchg_i64: 950 %pair = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic 951 %old = extractvalue { i64, i1 } %pair, 0 952 953 ; CHECK-NOT: dmb 954 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 955 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 956 957 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]: 958 ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]] 959 ; w0 below is a reasonable guess but could change: it certainly comes into the 960 ; function there. 961 ; CHECK-NEXT: cmp x[[OLD]], x0 962 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]] 963 ; As above, w1 is a reasonable guess. 964 ; CHECK: stxr [[STATUS:w[0-9]+]], x1, [x[[ADDR]]] 965 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]] 966 ; CHECK-NOT: dmb 967 968 ; CHECK: str x[[OLD]], 969 store i64 %old, i64* @var64 970 ret void 971 } 972 973 define i8 @test_atomic_load_monotonic_i8() nounwind { 974 ; CHECK-LABEL: test_atomic_load_monotonic_i8: 975 %val = load atomic i8, i8* @var8 monotonic, align 1 976 ; CHECK-NOT: dmb 977 ; CHECK: adrp x[[HIADDR:[0-9]+]], var8 978 ; CHECK: ldrb w0, [x[[HIADDR]], {{#?}}:lo12:var8] 979 ; CHECK-NOT: dmb 980 981 ret i8 %val 982 } 983 984 define i8 @test_atomic_load_monotonic_regoff_i8(i64 %base, i64 %off) nounwind { 985 ; CHECK-LABEL: test_atomic_load_monotonic_regoff_i8: 986 %addr_int = add i64 %base, %off 987 %addr = inttoptr i64 %addr_int to i8* 988 989 %val = load atomic i8, i8* %addr monotonic, align 1 990 ; CHECK-NOT: dmb 991 ; CHECK: ldrb w0, [x0, x1] 992 ; CHECK-NOT: dmb 993 994 ret i8 %val 995 } 996 997 define i8 @test_atomic_load_acquire_i8() nounwind { 998 ; CHECK-LABEL: test_atomic_load_acquire_i8: 999 %val = load atomic i8, i8* @var8 acquire, align 1 1000 ; CHECK-NOT: dmb 1001 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 1002 ; CHECK-NOT: dmb 1003 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 1004 ; CHECK-NOT: dmb 1005 ; CHECK: ldarb w0, [x[[ADDR]]] 1006 ; CHECK-NOT: dmb 1007 ret i8 %val 1008 } 1009 1010 define i8 @test_atomic_load_seq_cst_i8() nounwind { 1011 ; CHECK-LABEL: test_atomic_load_seq_cst_i8: 1012 %val = load atomic i8, i8* @var8 seq_cst, align 1 1013 ; CHECK-NOT: dmb 1014 ; CHECK: adrp [[HIADDR:x[0-9]+]], var8 1015 ; CHECK-NOT: dmb 1016 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var8 1017 ; CHECK-NOT: dmb 1018 ; CHECK: ldarb w0, [x[[ADDR]]] 1019 ; CHECK-NOT: dmb 1020 ret i8 %val 1021 } 1022 1023 define i16 @test_atomic_load_monotonic_i16() nounwind { 1024 ; CHECK-LABEL: test_atomic_load_monotonic_i16: 1025 %val = load atomic i16, i16* @var16 monotonic, align 2 1026 ; CHECK-NOT: dmb 1027 ; CHECK: adrp x[[HIADDR:[0-9]+]], var16 1028 ; CHECK-NOT: dmb 1029 ; CHECK: ldrh w0, [x[[HIADDR]], {{#?}}:lo12:var16] 1030 ; CHECK-NOT: dmb 1031 1032 ret i16 %val 1033 } 1034 1035 define i32 @test_atomic_load_monotonic_regoff_i32(i64 %base, i64 %off) nounwind { 1036 ; CHECK-LABEL: test_atomic_load_monotonic_regoff_i32: 1037 %addr_int = add i64 %base, %off 1038 %addr = inttoptr i64 %addr_int to i32* 1039 1040 %val = load atomic i32, i32* %addr monotonic, align 4 1041 ; CHECK-NOT: dmb 1042 ; CHECK: ldr w0, [x0, x1] 1043 ; CHECK-NOT: dmb 1044 1045 ret i32 %val 1046 } 1047 1048 define i64 @test_atomic_load_seq_cst_i64() nounwind { 1049 ; CHECK-LABEL: test_atomic_load_seq_cst_i64: 1050 %val = load atomic i64, i64* @var64 seq_cst, align 8 1051 ; CHECK-NOT: dmb 1052 ; CHECK: adrp [[HIADDR:x[0-9]+]], var64 1053 ; CHECK-NOT: dmb 1054 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var64 1055 ; CHECK-NOT: dmb 1056 ; CHECK: ldar x0, [x[[ADDR]]] 1057 ; CHECK-NOT: dmb 1058 ret i64 %val 1059 } 1060 1061 define void @test_atomic_store_monotonic_i8(i8 %val) nounwind { 1062 ; CHECK-LABEL: test_atomic_store_monotonic_i8: 1063 store atomic i8 %val, i8* @var8 monotonic, align 1 1064 ; CHECK: adrp x[[HIADDR:[0-9]+]], var8 1065 ; CHECK: strb w0, [x[[HIADDR]], {{#?}}:lo12:var8] 1066 1067 ret void 1068 } 1069 1070 define void @test_atomic_store_monotonic_regoff_i8(i64 %base, i64 %off, i8 %val) nounwind { 1071 ; CHECK-LABEL: test_atomic_store_monotonic_regoff_i8: 1072 1073 %addr_int = add i64 %base, %off 1074 %addr = inttoptr i64 %addr_int to i8* 1075 1076 store atomic i8 %val, i8* %addr monotonic, align 1 1077 ; CHECK: strb w2, [x0, x1] 1078 1079 ret void 1080 } 1081 define void @test_atomic_store_release_i8(i8 %val) nounwind { 1082 ; CHECK-LABEL: test_atomic_store_release_i8: 1083 store atomic i8 %val, i8* @var8 release, align 1 1084 ; CHECK-NOT: dmb 1085 ; CHECK: adrp [[HIADDR:x[0-9]+]], var8 1086 ; CHECK-NOT: dmb 1087 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var8 1088 ; CHECK-NOT: dmb 1089 ; CHECK: stlrb w0, [x[[ADDR]]] 1090 ; CHECK-NOT: dmb 1091 ret void 1092 } 1093 1094 define void @test_atomic_store_seq_cst_i8(i8 %val) nounwind { 1095 ; CHECK-LABEL: test_atomic_store_seq_cst_i8: 1096 store atomic i8 %val, i8* @var8 seq_cst, align 1 1097 ; CHECK-NOT: dmb 1098 ; CHECK: adrp [[HIADDR:x[0-9]+]], var8 1099 ; CHECK-NOT: dmb 1100 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var8 1101 ; CHECK-NOT: dmb 1102 ; CHECK: stlrb w0, [x[[ADDR]]] 1103 ; CHECK-NOT: dmb 1104 1105 ret void 1106 } 1107 1108 define void @test_atomic_store_monotonic_i16(i16 %val) nounwind { 1109 ; CHECK-LABEL: test_atomic_store_monotonic_i16: 1110 store atomic i16 %val, i16* @var16 monotonic, align 2 1111 ; CHECK-NOT: dmb 1112 ; CHECK: adrp x[[HIADDR:[0-9]+]], var16 1113 ; CHECK-NOT: dmb 1114 ; CHECK: strh w0, [x[[HIADDR]], {{#?}}:lo12:var16] 1115 ; CHECK-NOT: dmb 1116 ret void 1117 } 1118 1119 define void @test_atomic_store_monotonic_regoff_i32(i64 %base, i64 %off, i32 %val) nounwind { 1120 ; CHECK-LABEL: test_atomic_store_monotonic_regoff_i32: 1121 1122 %addr_int = add i64 %base, %off 1123 %addr = inttoptr i64 %addr_int to i32* 1124 1125 store atomic i32 %val, i32* %addr monotonic, align 4 1126 ; CHECK-NOT: dmb 1127 ; CHECK: str w2, [x0, x1] 1128 ; CHECK-NOT: dmb 1129 1130 ret void 1131 } 1132 1133 define void @test_atomic_store_release_i64(i64 %val) nounwind { 1134 ; CHECK-LABEL: test_atomic_store_release_i64: 1135 store atomic i64 %val, i64* @var64 release, align 8 1136 ; CHECK-NOT: dmb 1137 ; CHECK: adrp [[HIADDR:x[0-9]+]], var64 1138 ; CHECK-NOT: dmb 1139 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var64 1140 ; CHECK-NOT: dmb 1141 ; CHECK: stlr x0, [x[[ADDR]]] 1142 ; CHECK-NOT: dmb 1143 ret void 1144 } 1145