Home | History | Annotate | Download | only in AArch64
      1 ; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s
      2 
      3 @var8 = global i8 0
      4 @var16 = global i16 0
      5 @var32 = global i32 0
      6 @var64 = global i64 0
      7 
      8 define i8 @test_atomic_load_add_i8(i8 %offset) nounwind {
      9 ; CHECK: test_atomic_load_add_i8:
     10    %old = atomicrmw add i8* @var8, i8 %offset seq_cst
     11 ; CHECK: dmb ish
     12 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
     13 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
     14 
     15 ; CHECK: .LBB{{[0-9]+}}_1:
     16 ; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
     17   ; w0 below is a reasonable guess but could change: it certainly comes into the
     18   ;  function there.
     19 ; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
     20 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
     21 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
     22 ; CHECK: dmb ish
     23 
     24 ; CHECK: mov x0, x[[OLD]]
     25    ret i8 %old
     26 }
     27 
     28 define i16 @test_atomic_load_add_i16(i16 %offset) nounwind {
     29 ; CHECK: test_atomic_load_add_i16:
     30    %old = atomicrmw add i16* @var16, i16 %offset seq_cst
     31 ; CHECK: dmb ish
     32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
     33 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
     34 
     35 ; CHECK: .LBB{{[0-9]+}}_1:
     36 ; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
     37   ; w0 below is a reasonable guess but could change: it certainly comes into the
     38   ;  function there.
     39 ; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
     40 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
     41 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
     42 ; CHECK: dmb ish
     43 
     44 ; CHECK: mov x0, x[[OLD]]
     45    ret i16 %old
     46 }
     47 
     48 define i32 @test_atomic_load_add_i32(i32 %offset) nounwind {
     49 ; CHECK: test_atomic_load_add_i32:
     50    %old = atomicrmw add i32* @var32, i32 %offset seq_cst
     51 ; CHECK: dmb ish
     52 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
     53 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
     54 
     55 ; CHECK: .LBB{{[0-9]+}}_1:
     56 ; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
     57   ; w0 below is a reasonable guess but could change: it certainly comes into the
     58   ;  function there.
     59 ; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
     60 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
     61 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
     62 ; CHECK: dmb ish
     63 
     64 ; CHECK: mov x0, x[[OLD]]
     65    ret i32 %old
     66 }
     67 
     68 define i64 @test_atomic_load_add_i64(i64 %offset) nounwind {
     69 ; CHECK: test_atomic_load_add_i64:
     70    %old = atomicrmw add i64* @var64, i64 %offset seq_cst
     71 ; CHECK: dmb ish
     72 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
     73 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
     74 
     75 ; CHECK: .LBB{{[0-9]+}}_1:
     76 ; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
     77   ; x0 below is a reasonable guess but could change: it certainly comes into the
     78   ; function there.
     79 ; CHECK-NEXT: add [[NEW:x[0-9]+]], x[[OLD]], x0
     80 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
     81 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
     82 ; CHECK: dmb ish
     83 
     84 ; CHECK: mov x0, x[[OLD]]
     85    ret i64 %old
     86 }
     87 
     88 define i8 @test_atomic_load_sub_i8(i8 %offset) nounwind {
     89 ; CHECK: test_atomic_load_sub_i8:
     90    %old = atomicrmw sub i8* @var8, i8 %offset seq_cst
     91 ; CHECK: dmb ish
     92 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
     93 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
     94 
     95 ; CHECK: .LBB{{[0-9]+}}_1:
     96 ; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
     97   ; w0 below is a reasonable guess but could change: it certainly comes into the
     98   ;  function there.
     99 ; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
    100 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    101 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    102 ; CHECK: dmb ish
    103 
    104 ; CHECK: mov x0, x[[OLD]]
    105    ret i8 %old
    106 }
    107 
    108 define i16 @test_atomic_load_sub_i16(i16 %offset) nounwind {
    109 ; CHECK: test_atomic_load_sub_i16:
    110    %old = atomicrmw sub i16* @var16, i16 %offset seq_cst
    111 ; CHECK: dmb ish
    112 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    113 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    114 
    115 ; CHECK: .LBB{{[0-9]+}}_1:
    116 ; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    117   ; w0 below is a reasonable guess but could change: it certainly comes into the
    118   ;  function there.
    119 ; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
    120 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    121 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    122 ; CHECK: dmb ish
    123 
    124 ; CHECK: mov x0, x[[OLD]]
    125    ret i16 %old
    126 }
    127 
    128 define i32 @test_atomic_load_sub_i32(i32 %offset) nounwind {
    129 ; CHECK: test_atomic_load_sub_i32:
    130    %old = atomicrmw sub i32* @var32, i32 %offset seq_cst
    131 ; CHECK: dmb ish
    132 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    133 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    134 
    135 ; CHECK: .LBB{{[0-9]+}}_1:
    136 ; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    137   ; w0 below is a reasonable guess but could change: it certainly comes into the
    138   ;  function there.
    139 ; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
    140 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    141 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    142 ; CHECK: dmb ish
    143 
    144 ; CHECK: mov x0, x[[OLD]]
    145    ret i32 %old
    146 }
    147 
    148 define i64 @test_atomic_load_sub_i64(i64 %offset) nounwind {
    149 ; CHECK: test_atomic_load_sub_i64:
    150    %old = atomicrmw sub i64* @var64, i64 %offset seq_cst
    151 ; CHECK: dmb ish
    152 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    153 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    154 
    155 ; CHECK: .LBB{{[0-9]+}}_1:
    156 ; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    157   ; x0 below is a reasonable guess but could change: it certainly comes into the
    158   ; function there.
    159 ; CHECK-NEXT: sub [[NEW:x[0-9]+]], x[[OLD]], x0
    160 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    161 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    162 ; CHECK: dmb ish
    163 
    164 ; CHECK: mov x0, x[[OLD]]
    165    ret i64 %old
    166 }
    167 
    168 define i8 @test_atomic_load_and_i8(i8 %offset) nounwind {
    169 ; CHECK: test_atomic_load_and_i8:
    170    %old = atomicrmw and i8* @var8, i8 %offset seq_cst
    171 ; CHECK: dmb ish
    172 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    173 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    174 
    175 ; CHECK: .LBB{{[0-9]+}}_1:
    176 ; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    177   ; w0 below is a reasonable guess but could change: it certainly comes into the
    178   ;  function there.
    179 ; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
    180 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    181 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    182 ; CHECK: dmb ish
    183 
    184 ; CHECK: mov x0, x[[OLD]]
    185    ret i8 %old
    186 }
    187 
    188 define i16 @test_atomic_load_and_i16(i16 %offset) nounwind {
    189 ; CHECK: test_atomic_load_and_i16:
    190    %old = atomicrmw and i16* @var16, i16 %offset seq_cst
    191 ; CHECK: dmb ish
    192 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    193 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    194 
    195 ; CHECK: .LBB{{[0-9]+}}_1:
    196 ; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    197   ; w0 below is a reasonable guess but could change: it certainly comes into the
    198   ;  function there.
    199 ; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
    200 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    201 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    202 ; CHECK: dmb ish
    203 
    204 ; CHECK: mov x0, x[[OLD]]
    205    ret i16 %old
    206 }
    207 
    208 define i32 @test_atomic_load_and_i32(i32 %offset) nounwind {
    209 ; CHECK: test_atomic_load_and_i32:
    210    %old = atomicrmw and i32* @var32, i32 %offset seq_cst
    211 ; CHECK: dmb ish
    212 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    213 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    214 
    215 ; CHECK: .LBB{{[0-9]+}}_1:
    216 ; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    217   ; w0 below is a reasonable guess but could change: it certainly comes into the
    218   ;  function there.
    219 ; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
    220 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    221 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    222 ; CHECK: dmb ish
    223 
    224 ; CHECK: mov x0, x[[OLD]]
    225    ret i32 %old
    226 }
    227 
    228 define i64 @test_atomic_load_and_i64(i64 %offset) nounwind {
    229 ; CHECK: test_atomic_load_and_i64:
    230    %old = atomicrmw and i64* @var64, i64 %offset seq_cst
    231 ; CHECK: dmb ish
    232 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    233 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    234 
    235 ; CHECK: .LBB{{[0-9]+}}_1:
    236 ; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    237   ; x0 below is a reasonable guess but could change: it certainly comes into the
    238   ; function there.
    239 ; CHECK-NEXT: and [[NEW:x[0-9]+]], x[[OLD]], x0
    240 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    241 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    242 ; CHECK: dmb ish
    243 
    244 ; CHECK: mov x0, x[[OLD]]
    245    ret i64 %old
    246 }
    247 
    248 define i8 @test_atomic_load_or_i8(i8 %offset) nounwind {
    249 ; CHECK: test_atomic_load_or_i8:
    250    %old = atomicrmw or i8* @var8, i8 %offset seq_cst
    251 ; CHECK: dmb ish
    252 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    253 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    254 
    255 ; CHECK: .LBB{{[0-9]+}}_1:
    256 ; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    257   ; w0 below is a reasonable guess but could change: it certainly comes into the
    258   ;  function there.
    259 ; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
    260 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    261 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    262 ; CHECK: dmb ish
    263 
    264 ; CHECK: mov x0, x[[OLD]]
    265    ret i8 %old
    266 }
    267 
    268 define i16 @test_atomic_load_or_i16(i16 %offset) nounwind {
    269 ; CHECK: test_atomic_load_or_i16:
    270    %old = atomicrmw or i16* @var16, i16 %offset seq_cst
    271 ; CHECK: dmb ish
    272 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    273 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    274 
    275 ; CHECK: .LBB{{[0-9]+}}_1:
    276 ; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    277   ; w0 below is a reasonable guess but could change: it certainly comes into the
    278   ;  function there.
    279 ; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
    280 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    281 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    282 ; CHECK: dmb ish
    283 
    284 ; CHECK: mov x0, x[[OLD]]
    285    ret i16 %old
    286 }
    287 
    288 define i32 @test_atomic_load_or_i32(i32 %offset) nounwind {
    289 ; CHECK: test_atomic_load_or_i32:
    290    %old = atomicrmw or i32* @var32, i32 %offset seq_cst
    291 ; CHECK: dmb ish
    292 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    293 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    294 
    295 ; CHECK: .LBB{{[0-9]+}}_1:
    296 ; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    297   ; w0 below is a reasonable guess but could change: it certainly comes into the
    298   ;  function there.
    299 ; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
    300 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    301 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    302 ; CHECK: dmb ish
    303 
    304 ; CHECK: mov x0, x[[OLD]]
    305    ret i32 %old
    306 }
    307 
    308 define i64 @test_atomic_load_or_i64(i64 %offset) nounwind {
    309 ; CHECK: test_atomic_load_or_i64:
    310    %old = atomicrmw or i64* @var64, i64 %offset seq_cst
    311 ; CHECK: dmb ish
    312 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    313 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    314 
    315 ; CHECK: .LBB{{[0-9]+}}_1:
    316 ; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    317   ; x0 below is a reasonable guess but could change: it certainly comes into the
    318   ; function there.
    319 ; CHECK-NEXT: orr [[NEW:x[0-9]+]], x[[OLD]], x0
    320 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    321 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    322 ; CHECK: dmb ish
    323 
    324 ; CHECK: mov x0, x[[OLD]]
    325    ret i64 %old
    326 }
    327 
    328 define i8 @test_atomic_load_xor_i8(i8 %offset) nounwind {
    329 ; CHECK: test_atomic_load_xor_i8:
    330    %old = atomicrmw xor i8* @var8, i8 %offset seq_cst
    331 ; CHECK: dmb ish
    332 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    333 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    334 
    335 ; CHECK: .LBB{{[0-9]+}}_1:
    336 ; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    337   ; w0 below is a reasonable guess but could change: it certainly comes into the
    338   ;  function there.
    339 ; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
    340 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    341 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    342 ; CHECK: dmb ish
    343 
    344 ; CHECK: mov x0, x[[OLD]]
    345    ret i8 %old
    346 }
    347 
    348 define i16 @test_atomic_load_xor_i16(i16 %offset) nounwind {
    349 ; CHECK: test_atomic_load_xor_i16:
    350    %old = atomicrmw xor i16* @var16, i16 %offset seq_cst
    351 ; CHECK: dmb ish
    352 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    353 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    354 
    355 ; CHECK: .LBB{{[0-9]+}}_1:
    356 ; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    357   ; w0 below is a reasonable guess but could change: it certainly comes into the
    358   ;  function there.
    359 ; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
    360 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    361 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    362 ; CHECK: dmb ish
    363 
    364 ; CHECK: mov x0, x[[OLD]]
    365    ret i16 %old
    366 }
    367 
    368 define i32 @test_atomic_load_xor_i32(i32 %offset) nounwind {
    369 ; CHECK: test_atomic_load_xor_i32:
    370    %old = atomicrmw xor i32* @var32, i32 %offset seq_cst
    371 ; CHECK: dmb ish
    372 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    373 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    374 
    375 ; CHECK: .LBB{{[0-9]+}}_1:
    376 ; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    377   ; w0 below is a reasonable guess but could change: it certainly comes into the
    378   ;  function there.
    379 ; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
    380 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    381 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    382 ; CHECK: dmb ish
    383 
    384 ; CHECK: mov x0, x[[OLD]]
    385    ret i32 %old
    386 }
    387 
    388 define i64 @test_atomic_load_xor_i64(i64 %offset) nounwind {
    389 ; CHECK: test_atomic_load_xor_i64:
    390    %old = atomicrmw xor i64* @var64, i64 %offset seq_cst
    391 ; CHECK: dmb ish
    392 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    393 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    394 
    395 ; CHECK: .LBB{{[0-9]+}}_1:
    396 ; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    397   ; x0 below is a reasonable guess but could change: it certainly comes into the
    398   ; function there.
    399 ; CHECK-NEXT: eor [[NEW:x[0-9]+]], x[[OLD]], x0
    400 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    401 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    402 ; CHECK: dmb ish
    403 
    404 ; CHECK: mov x0, x[[OLD]]
    405    ret i64 %old
    406 }
    407 
    408 define i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
    409 ; CHECK: test_atomic_load_xchg_i8:
    410    %old = atomicrmw xchg i8* @var8, i8 %offset seq_cst
    411 ; CHECK: dmb ish
    412 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    413 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    414 
    415 ; CHECK: .LBB{{[0-9]+}}_1:
    416 ; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    417   ; w0 below is a reasonable guess but could change: it certainly comes into the
    418   ; function there.
    419 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
    420 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    421 ; CHECK: dmb ish
    422 
    423 ; CHECK: mov x0, x[[OLD]]
    424    ret i8 %old
    425 }
    426 
    427 define i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind {
    428 ; CHECK: test_atomic_load_xchg_i16:
    429    %old = atomicrmw xchg i16* @var16, i16 %offset seq_cst
    430 ; CHECK: dmb ish
    431 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    432 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    433 
    434 ; CHECK: .LBB{{[0-9]+}}_1:
    435 ; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    436   ; w0 below is a reasonable guess but could change: it certainly comes into the
    437   ; function there.
    438 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
    439 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    440 ; CHECK: dmb ish
    441 
    442 ; CHECK: mov x0, x[[OLD]]
    443    ret i16 %old
    444 }
    445 
    446 define i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind {
    447 ; CHECK: test_atomic_load_xchg_i32:
    448    %old = atomicrmw xchg i32* @var32, i32 %offset seq_cst
    449 ; CHECK: dmb ish
    450 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    451 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    452 
    453 ; CHECK: .LBB{{[0-9]+}}_1:
    454 ; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    455   ; w0 below is a reasonable guess but could change: it certainly comes into the
    456   ;  function there.
    457 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
    458 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    459 ; CHECK: dmb ish
    460 
    461 ; CHECK: mov x0, x[[OLD]]
    462    ret i32 %old
    463 }
    464 
    465 define i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind {
    466 ; CHECK: test_atomic_load_xchg_i64:
    467    %old = atomicrmw xchg i64* @var64, i64 %offset seq_cst
    468 ; CHECK: dmb ish
    469 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    470 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    471 
    472 ; CHECK: .LBB{{[0-9]+}}_1:
    473 ; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    474   ; x0 below is a reasonable guess but could change: it certainly comes into the
    475   ; function there.
    476 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], x0, [x[[ADDR]]]
    477 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    478 ; CHECK: dmb ish
    479 
    480 ; CHECK: mov x0, x[[OLD]]
    481    ret i64 %old
    482 }
    483 
    484 
    485 define i8 @test_atomic_load_min_i8(i8 %offset) nounwind {
    486 ; CHECK: test_atomic_load_min_i8:
    487    %old = atomicrmw min i8* @var8, i8 %offset seq_cst
    488 ; CHECK: dmb ish
    489 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    490 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    491 
    492 ; CHECK: .LBB{{[0-9]+}}_1:
    493 ; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    494   ; w0 below is a reasonable guess but could change: it certainly comes into the
    495   ;  function there.
    496 ; CHECK-NEXT: cmp w0, w[[OLD]], sxtb
    497 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
    498 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    499 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    500 ; CHECK: dmb ish
    501 
    502 ; CHECK: mov x0, x[[OLD]]
    503    ret i8 %old
    504 }
    505 
    506 define i16 @test_atomic_load_min_i16(i16 %offset) nounwind {
    507 ; CHECK: test_atomic_load_min_i16:
    508    %old = atomicrmw min i16* @var16, i16 %offset seq_cst
    509 ; CHECK: dmb ish
    510 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    511 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    512 
    513 ; CHECK: .LBB{{[0-9]+}}_1:
    514 ; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    515   ; w0 below is a reasonable guess but could change: it certainly comes into the
    516   ;  function there.
    517 ; CHECK-NEXT: cmp w0, w[[OLD]], sxth
    518 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
    519 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    520 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    521 ; CHECK: dmb ish
    522 
    523 ; CHECK: mov x0, x[[OLD]]
    524    ret i16 %old
    525 }
    526 
    527 define i32 @test_atomic_load_min_i32(i32 %offset) nounwind {
    528 ; CHECK: test_atomic_load_min_i32:
    529    %old = atomicrmw min i32* @var32, i32 %offset seq_cst
    530 ; CHECK: dmb ish
    531 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    532 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    533 
    534 ; CHECK: .LBB{{[0-9]+}}_1:
    535 ; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    536   ; w0 below is a reasonable guess but could change: it certainly comes into the
    537   ;  function there.
    538 ; CHECK-NEXT: cmp w0, w[[OLD]]
    539 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
    540 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    541 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    542 ; CHECK: dmb ish
    543 
    544 ; CHECK: mov x0, x[[OLD]]
    545    ret i32 %old
    546 }
    547 
    548 define i64 @test_atomic_load_min_i64(i64 %offset) nounwind {
    549 ; CHECK: test_atomic_load_min_i64:
    550    %old = atomicrmw min i64* @var64, i64 %offset seq_cst
    551 ; CHECK: dmb ish
    552 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    553 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    554 
    555 ; CHECK: .LBB{{[0-9]+}}_1:
    556 ; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    557   ; x0 below is a reasonable guess but could change: it certainly comes into the
    558   ; function there.
    559 ; CHECK-NEXT: cmp x0, x[[OLD]]
    560 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, gt
    561 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    562 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    563 ; CHECK: dmb ish
    564 
    565 ; CHECK: mov x0, x[[OLD]]
    566    ret i64 %old
    567 }
    568 
    569 define i8 @test_atomic_load_max_i8(i8 %offset) nounwind {
    570 ; CHECK: test_atomic_load_max_i8:
    571    %old = atomicrmw max i8* @var8, i8 %offset seq_cst
    572 ; CHECK: dmb ish
    573 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    574 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    575 
    576 ; CHECK: .LBB{{[0-9]+}}_1:
    577 ; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    578   ; w0 below is a reasonable guess but could change: it certainly comes into the
    579   ;  function there.
    580 ; CHECK-NEXT: cmp w0, w[[OLD]], sxtb
    581 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lt
    582 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    583 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    584 ; CHECK: dmb ish
    585 
    586 ; CHECK: mov x0, x[[OLD]]
    587    ret i8 %old
    588 }
    589 
    590 define i16 @test_atomic_load_max_i16(i16 %offset) nounwind {
    591 ; CHECK: test_atomic_load_max_i16:
    592    %old = atomicrmw max i16* @var16, i16 %offset seq_cst
    593 ; CHECK: dmb ish
    594 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    595 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    596 
    597 ; CHECK: .LBB{{[0-9]+}}_1:
    598 ; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    599   ; w0 below is a reasonable guess but could change: it certainly comes into the
    600   ;  function there.
    601 ; CHECK-NEXT: cmp w0, w[[OLD]], sxth
    602 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lt
    603 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    604 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    605 ; CHECK: dmb ish
    606 
    607 ; CHECK: mov x0, x[[OLD]]
    608    ret i16 %old
    609 }
    610 
    611 define i32 @test_atomic_load_max_i32(i32 %offset) nounwind {
    612 ; CHECK: test_atomic_load_max_i32:
    613    %old = atomicrmw max i32* @var32, i32 %offset seq_cst
    614 ; CHECK: dmb ish
    615 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    616 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    617 
    618 ; CHECK: .LBB{{[0-9]+}}_1:
    619 ; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    620   ; w0 below is a reasonable guess but could change: it certainly comes into the
    621   ;  function there.
    622 ; CHECK-NEXT: cmp w0, w[[OLD]]
    623 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lt
    624 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    625 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    626 ; CHECK: dmb ish
    627 
    628 ; CHECK: mov x0, x[[OLD]]
    629    ret i32 %old
    630 }
    631 
    632 define i64 @test_atomic_load_max_i64(i64 %offset) nounwind {
    633 ; CHECK: test_atomic_load_max_i64:
    634    %old = atomicrmw max i64* @var64, i64 %offset seq_cst
    635 ; CHECK: dmb ish
    636 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    637 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    638 
    639 ; CHECK: .LBB{{[0-9]+}}_1:
    640 ; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    641   ; x0 below is a reasonable guess but could change: it certainly comes into the
    642   ; function there.
    643 ; CHECK-NEXT: cmp x0, x[[OLD]]
    644 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, lt
    645 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    646 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    647 ; CHECK: dmb ish
    648 
    649 ; CHECK: mov x0, x[[OLD]]
    650    ret i64 %old
    651 }
    652 
    653 define i8 @test_atomic_load_umin_i8(i8 %offset) nounwind {
    654 ; CHECK: test_atomic_load_umin_i8:
    655    %old = atomicrmw umin i8* @var8, i8 %offset seq_cst
    656 ; CHECK: dmb ish
    657 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    658 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    659 
    660 ; CHECK: .LBB{{[0-9]+}}_1:
    661 ; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    662   ; w0 below is a reasonable guess but could change: it certainly comes into the
    663   ;  function there.
    664 ; CHECK-NEXT: cmp w0, w[[OLD]], uxtb
    665 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
    666 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    667 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    668 ; CHECK: dmb ish
    669 
    670 ; CHECK: mov x0, x[[OLD]]
    671    ret i8 %old
    672 }
    673 
    674 define i16 @test_atomic_load_umin_i16(i16 %offset) nounwind {
    675 ; CHECK: test_atomic_load_umin_i16:
    676    %old = atomicrmw umin i16* @var16, i16 %offset seq_cst
    677 ; CHECK: dmb ish
    678 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    679 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    680 
    681 ; CHECK: .LBB{{[0-9]+}}_1:
    682 ; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    683   ; w0 below is a reasonable guess but could change: it certainly comes into the
    684   ;  function there.
    685 ; CHECK-NEXT: cmp w0, w[[OLD]], uxth
    686 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
    687 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    688 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    689 ; CHECK: dmb ish
    690 
    691 ; CHECK: mov x0, x[[OLD]]
    692    ret i16 %old
    693 }
    694 
    695 define i32 @test_atomic_load_umin_i32(i32 %offset) nounwind {
    696 ; CHECK: test_atomic_load_umin_i32:
    697    %old = atomicrmw umin i32* @var32, i32 %offset seq_cst
    698 ; CHECK: dmb ish
    699 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    700 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    701 
    702 ; CHECK: .LBB{{[0-9]+}}_1:
    703 ; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    704   ; w0 below is a reasonable guess but could change: it certainly comes into the
    705   ;  function there.
    706 ; CHECK-NEXT: cmp w0, w[[OLD]]
    707 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
    708 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    709 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    710 ; CHECK: dmb ish
    711 
    712 ; CHECK: mov x0, x[[OLD]]
    713    ret i32 %old
    714 }
    715 
    716 define i64 @test_atomic_load_umin_i64(i64 %offset) nounwind {
    717 ; CHECK: test_atomic_load_umin_i64:
    718    %old = atomicrmw umin i64* @var64, i64 %offset seq_cst
    719 ; CHECK: dmb ish
    720 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    721 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    722 
    723 ; CHECK: .LBB{{[0-9]+}}_1:
    724 ; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    725   ; x0 below is a reasonable guess but could change: it certainly comes into the
    726   ; function there.
    727 ; CHECK-NEXT: cmp x0, x[[OLD]]
    728 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, hi
    729 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    730 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    731 ; CHECK: dmb ish
    732 
    733 ; CHECK: mov x0, x[[OLD]]
    734    ret i64 %old
    735 }
    736 
    737 define i8 @test_atomic_load_umax_i8(i8 %offset) nounwind {
    738 ; CHECK: test_atomic_load_umax_i8:
    739    %old = atomicrmw umax i8* @var8, i8 %offset seq_cst
    740 ; CHECK: dmb ish
    741 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    742 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    743 
    744 ; CHECK: .LBB{{[0-9]+}}_1:
    745 ; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    746   ; w0 below is a reasonable guess but could change: it certainly comes into the
    747   ;  function there.
    748 ; CHECK-NEXT: cmp w0, w[[OLD]], uxtb
    749 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lo
    750 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    751 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    752 ; CHECK: dmb ish
    753 
    754 ; CHECK: mov x0, x[[OLD]]
    755    ret i8 %old
    756 }
    757 
    758 define i16 @test_atomic_load_umax_i16(i16 %offset) nounwind {
    759 ; CHECK: test_atomic_load_umax_i16:
    760    %old = atomicrmw umax i16* @var16, i16 %offset seq_cst
    761 ; CHECK: dmb ish
    762 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    763 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    764 
    765 ; CHECK: .LBB{{[0-9]+}}_1:
    766 ; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    767   ; w0 below is a reasonable guess but could change: it certainly comes into the
    768   ;  function there.
    769 ; CHECK-NEXT: cmp w0, w[[OLD]], uxth
    770 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lo
    771 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    772 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    773 ; CHECK: dmb ish
    774 
    775 ; CHECK: mov x0, x[[OLD]]
    776    ret i16 %old
    777 }
    778 
    779 define i32 @test_atomic_load_umax_i32(i32 %offset) nounwind {
    780 ; CHECK: test_atomic_load_umax_i32:
    781    %old = atomicrmw umax i32* @var32, i32 %offset seq_cst
    782 ; CHECK: dmb ish
    783 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    784 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    785 
    786 ; CHECK: .LBB{{[0-9]+}}_1:
    787 ; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    788   ; w0 below is a reasonable guess but could change: it certainly comes into the
    789   ;  function there.
    790 ; CHECK-NEXT: cmp w0, w[[OLD]]
    791 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lo
    792 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    793 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    794 ; CHECK: dmb ish
    795 
    796 ; CHECK: mov x0, x[[OLD]]
    797    ret i32 %old
    798 }
    799 
    800 define i64 @test_atomic_load_umax_i64(i64 %offset) nounwind {
    801 ; CHECK: test_atomic_load_umax_i64:
    802    %old = atomicrmw umax i64* @var64, i64 %offset seq_cst
    803 ; CHECK: dmb ish
    804 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    805 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    806 
    807 ; CHECK: .LBB{{[0-9]+}}_1:
    808 ; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    809   ; x0 below is a reasonable guess but could change: it certainly comes into the
    810   ; function there.
    811 ; CHECK-NEXT: cmp x0, x[[OLD]]
    812 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, lo
    813 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    814 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    815 ; CHECK: dmb ish
    816 
    817 ; CHECK: mov x0, x[[OLD]]
    818    ret i64 %old
    819 }
    820 
    821 define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
    822 ; CHECK: test_atomic_cmpxchg_i8:
    823    %old = cmpxchg i8* @var8, i8 %wanted, i8 %new seq_cst
    824 ; CHECK: dmb ish
    825 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    826 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    827 
    828 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
    829 ; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    830   ; w0 below is a reasonable guess but could change: it certainly comes into the
    831   ;  function there.
    832 ; CHECK-NEXT: cmp w[[OLD]], w0
    833 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
    834   ; As above, w1 is a reasonable guess.
    835 ; CHECK: stxrb [[STATUS:w[0-9]+]], w1, [x[[ADDR]]]
    836 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
    837 ; CHECK: dmb ish
    838 
    839 ; CHECK: mov x0, x[[OLD]]
    840    ret i8 %old
    841 }
    842 
    843 define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
    844 ; CHECK: test_atomic_cmpxchg_i16:
    845    %old = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst
    846 ; CHECK: dmb ish
    847 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    848 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    849 
    850 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
    851 ; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    852   ; w0 below is a reasonable guess but could change: it certainly comes into the
    853   ;  function there.
    854 ; CHECK-NEXT: cmp w[[OLD]], w0
    855 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
    856   ; As above, w1 is a reasonable guess.
    857 ; CHECK: stxrh [[STATUS:w[0-9]+]], w1, [x[[ADDR]]]
    858 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
    859 ; CHECK: dmb ish
    860 
    861 ; CHECK: mov x0, x[[OLD]]
    862    ret i16 %old
    863 }
    864 
    865 define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
    866 ; CHECK: test_atomic_cmpxchg_i32:
    867    %old = cmpxchg i32* @var32, i32 %wanted, i32 %new seq_cst
    868 ; CHECK: dmb ish
    869 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    870 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    871 
    872 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
    873 ; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    874   ; w0 below is a reasonable guess but could change: it certainly comes into the
    875   ;  function there.
    876 ; CHECK-NEXT: cmp w[[OLD]], w0
    877 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
    878   ; As above, w1 is a reasonable guess.
    879 ; CHECK: stxr [[STATUS:w[0-9]+]], w1, [x[[ADDR]]]
    880 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
    881 ; CHECK: dmb ish
    882 
    883 ; CHECK: mov x0, x[[OLD]]
    884    ret i32 %old
    885 }
    886 
    887 define i64 @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
    888 ; CHECK: test_atomic_cmpxchg_i64:
    889    %old = cmpxchg i64* @var64, i64 %wanted, i64 %new seq_cst
    890 ; CHECK: dmb ish
    891 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    892 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    893 
    894 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
    895 ; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    896   ; w0 below is a reasonable guess but could change: it certainly comes into the
    897   ;  function there.
    898 ; CHECK-NEXT: cmp x[[OLD]], x0
    899 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
    900   ; As above, w1 is a reasonable guess.
    901 ; CHECK: stxr [[STATUS:w[0-9]+]], x1, [x[[ADDR]]]
    902 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
    903 ; CHECK: dmb ish
    904 
    905 ; CHECK: mov x0, x[[OLD]]
    906    ret i64 %old
    907 }
    908 
    909 define i8 @test_atomic_load_monotonic_i8() nounwind {
    910 ; CHECK: test_atomic_load_monotonic_i8:
    911   %val = load atomic i8* @var8 monotonic, align 1
    912 ; CHECK-NOT: dmb
    913 ; CHECK: adrp x[[HIADDR:[0-9]+]], var8
    914 ; CHECK: ldrb w0, [x[[HIADDR]], #:lo12:var8]
    915 ; CHECK-NOT: dmb
    916 
    917   ret i8 %val
    918 }
    919 
    920 define i8 @test_atomic_load_monotonic_regoff_i8(i64 %base, i64 %off) nounwind {
    921 ; CHECK: test_atomic_load_monotonic_regoff_i8:
    922   %addr_int = add i64 %base, %off
    923   %addr = inttoptr i64 %addr_int to i8*
    924 
    925   %val = load atomic i8* %addr monotonic, align 1
    926 ; CHECK-NOT: dmb
    927 ; CHECK: ldrb w0, [x0, x1]
    928 ; CHECK-NOT: dmb
    929 
    930   ret i8 %val
    931 }
    932 
    933 define i8 @test_atomic_load_acquire_i8() nounwind {
    934 ; CHECK: test_atomic_load_acquire_i8:
    935   %val = load atomic i8* @var8 acquire, align 1
    936 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    937 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    938 
    939 ; CHECK: ldarb w0, [x[[ADDR]]]
    940   ret i8 %val
    941 }
    942 
    943 define i8 @test_atomic_load_seq_cst_i8() nounwind {
    944 ; CHECK: test_atomic_load_seq_cst_i8:
    945   %val = load atomic i8* @var8 seq_cst, align 1
    946 ; CHECK: adrp x[[HIADDR:[0-9]+]], var8
    947 ; CHECK: ldrb w0, [x[[HIADDR]], #:lo12:var8]
    948 ; CHECK: dmb ish
    949   ret i8 %val
    950 }
    951 
    952 define i16 @test_atomic_load_monotonic_i16() nounwind {
    953 ; CHECK: test_atomic_load_monotonic_i16:
    954   %val = load atomic i16* @var16 monotonic, align 2
    955 ; CHECK-NOT: dmb
    956 ; CHECK: adrp x[[HIADDR:[0-9]+]], var16
    957 ; CHECK: ldrh w0, [x[[HIADDR]], #:lo12:var16]
    958 ; CHECK-NOT: dmb
    959 
    960   ret i16 %val
    961 }
    962 
    963 define i32 @test_atomic_load_monotonic_regoff_i32(i64 %base, i64 %off) nounwind {
    964 ; CHECK: test_atomic_load_monotonic_regoff_i32:
    965   %addr_int = add i64 %base, %off
    966   %addr = inttoptr i64 %addr_int to i32*
    967 
    968   %val = load atomic i32* %addr monotonic, align 4
    969 ; CHECK-NOT: dmb
    970 ; CHECK: ldr w0, [x0, x1]
    971 ; CHECK-NOT: dmb
    972 
    973   ret i32 %val
    974 }
    975 
    976 define i64 @test_atomic_load_seq_cst_i64() nounwind {
    977 ; CHECK: test_atomic_load_seq_cst_i64:
    978   %val = load atomic i64* @var64 seq_cst, align 8
    979 ; CHECK: adrp x[[HIADDR:[0-9]+]], var64
    980 ; CHECK: ldr x0, [x[[HIADDR]], #:lo12:var64]
    981 ; CHECK: dmb ish
    982   ret i64 %val
    983 }
    984 
    985 define void @test_atomic_store_monotonic_i8(i8 %val) nounwind {
    986 ; CHECK: test_atomic_store_monotonic_i8:
    987   store atomic i8 %val, i8* @var8 monotonic, align 1
    988 ; CHECK: adrp x[[HIADDR:[0-9]+]], var8
    989 ; CHECK: strb w0, [x[[HIADDR]], #:lo12:var8]
    990 
    991   ret void
    992 }
    993 
    994 define void @test_atomic_store_monotonic_regoff_i8(i64 %base, i64 %off, i8 %val) nounwind {
    995 ; CHECK: test_atomic_store_monotonic_regoff_i8:
    996 
    997   %addr_int = add i64 %base, %off
    998   %addr = inttoptr i64 %addr_int to i8*
    999 
   1000   store atomic i8 %val, i8* %addr monotonic, align 1
   1001 ; CHECK: strb w2, [x0, x1]
   1002 
   1003   ret void
   1004 }
   1005 define void @test_atomic_store_release_i8(i8 %val) nounwind {
   1006 ; CHECK: test_atomic_store_release_i8:
   1007   store atomic i8 %val, i8* @var8 release, align 1
   1008 ; CHECK: adrp [[HIADDR:x[0-9]+]], var8
   1009 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], #:lo12:var8
   1010 ; CHECK: stlrb w0, [x[[ADDR]]]
   1011 
   1012   ret void
   1013 }
   1014 
   1015 define void @test_atomic_store_seq_cst_i8(i8 %val) nounwind {
   1016 ; CHECK: test_atomic_store_seq_cst_i8:
   1017   store atomic i8 %val, i8* @var8 seq_cst, align 1
   1018 ; CHECK: adrp [[HIADDR:x[0-9]+]], var8
   1019 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], #:lo12:var8
   1020 ; CHECK: stlrb w0, [x[[ADDR]]]
   1021 ; CHECK: dmb ish
   1022 
   1023   ret void
   1024 }
   1025 
   1026 define void @test_atomic_store_monotonic_i16(i16 %val) nounwind {
   1027 ; CHECK: test_atomic_store_monotonic_i16:
   1028   store atomic i16 %val, i16* @var16 monotonic, align 2
   1029 ; CHECK: adrp x[[HIADDR:[0-9]+]], var16
   1030 ; CHECK: strh w0, [x[[HIADDR]], #:lo12:var16]
   1031 
   1032   ret void
   1033 }
   1034 
   1035 define void @test_atomic_store_monotonic_regoff_i32(i64 %base, i64 %off, i32 %val) nounwind {
   1036 ; CHECK: test_atomic_store_monotonic_regoff_i32:
   1037 
   1038   %addr_int = add i64 %base, %off
   1039   %addr = inttoptr i64 %addr_int to i32*
   1040 
   1041   store atomic i32 %val, i32* %addr monotonic, align 4
   1042 ; CHECK: str w2, [x0, x1]
   1043 
   1044   ret void
   1045 }
   1046 
   1047 define void @test_atomic_store_release_i64(i64 %val) nounwind {
   1048 ; CHECK: test_atomic_store_release_i64:
   1049   store atomic i64 %val, i64* @var64 release, align 8
   1050 ; CHECK: adrp [[HIADDR:x[0-9]+]], var64
   1051 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], #:lo12:var64
   1052 ; CHECK: stlr x0, [x[[ADDR]]]
   1053 
   1054   ret void
   1055 }
   1056