Home | History | Annotate | Download | only in AArch64 
      1 ; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs < %s | FileCheck %s
      2 ; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-REG
      3 
      4 
      5 ; Point of CHECK-REG is to make sure UNPREDICTABLE instructions aren't created
      6 ; (i.e. reusing a register for status & data in store exclusive).
      7 ; CHECK-REG-NOT: stlxrb w[[NEW:[0-9]+]], w[[NEW]], [x{{[0-9]+}}]
      8 ; CHECK-REG-NOT: stlxrb w[[NEW:[0-9]+]], x[[NEW]], [x{{[0-9]+}}]
      9 
     10 @var8 = global i8 0
     11 @var16 = global i16 0
     12 @var32 = global i32 0
     13 @var64 = global i64 0
     14 
     15 define i8 @test_atomic_load_add_i8(i8 %offset) nounwind {
     16 ; CHECK-LABEL: test_atomic_load_add_i8:
     17    %old = atomicrmw add i8* @var8, i8 %offset seq_cst
     18 ; CHECK-NOT: dmb
     19 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
     20 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
     21 
     22 ; CHECK: .LBB{{[0-9]+}}_1:
     23 ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
     24   ; w0 below is a reasonable guess but could change: it certainly comes into the
     25   ;  function there.
     26 ; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
     27 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
     28 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
     29 ; CHECK-NOT: dmb
     30 
     31 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
     32    ret i8 %old
     33 }
     34 
     35 define i16 @test_atomic_load_add_i16(i16 %offset) nounwind {
     36 ; CHECK-LABEL: test_atomic_load_add_i16:
     37    %old = atomicrmw add i16* @var16, i16 %offset acquire
     38 ; CHECK-NOT: dmb
     39 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
     40 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
     41 
     42 ; CHECK: .LBB{{[0-9]+}}_1:
     43 ; ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
     44   ; w0 below is a reasonable guess but could change: it certainly comes into the
     45   ;  function there.
     46 ; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
     47 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
     48 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
     49 ; CHECK-NOT: dmb
     50 
     51 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
     52    ret i16 %old
     53 }
     54 
     55 define i32 @test_atomic_load_add_i32(i32 %offset) nounwind {
     56 ; CHECK-LABEL: test_atomic_load_add_i32:
     57    %old = atomicrmw add i32* @var32, i32 %offset release
     58 ; CHECK-NOT: dmb
     59 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
     60 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
     61 
     62 ; CHECK: .LBB{{[0-9]+}}_1:
     63 ; ; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
     64   ; w0 below is a reasonable guess but could change: it certainly comes into the
     65   ;  function there.
     66 ; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
     67 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
     68 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
     69 ; CHECK-NOT: dmb
     70 
     71 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
     72    ret i32 %old
     73 }
     74 
     75 define i64 @test_atomic_load_add_i64(i64 %offset) nounwind {
     76 ; CHECK-LABEL: test_atomic_load_add_i64:
     77    %old = atomicrmw add i64* @var64, i64 %offset monotonic
     78 ; CHECK-NOT: dmb
     79 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
     80 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
     81 
     82 ; CHECK: .LBB{{[0-9]+}}_1:
     83 ; ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
     84   ; x0 below is a reasonable guess but could change: it certainly comes into the
     85   ; function there.
     86 ; CHECK-NEXT: add [[NEW:x[0-9]+]], x[[OLD]], x0
     87 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
     88 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
     89 ; CHECK-NOT: dmb
     90 
     91 ; CHECK: mov x0, x[[OLD]]
     92    ret i64 %old
     93 }
     94 
     95 define i8 @test_atomic_load_sub_i8(i8 %offset) nounwind {
     96 ; CHECK-LABEL: test_atomic_load_sub_i8:
     97    %old = atomicrmw sub i8* @var8, i8 %offset monotonic
     98 ; CHECK-NOT: dmb
     99 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    100 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    101 
    102 ; CHECK: .LBB{{[0-9]+}}_1:
    103 ; ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    104   ; w0 below is a reasonable guess but could change: it certainly comes into the
    105   ;  function there.
    106 ; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
    107 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    108 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    109 ; CHECK-NOT: dmb
    110 
    111 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    112    ret i8 %old
    113 }
    114 
    115 define i16 @test_atomic_load_sub_i16(i16 %offset) nounwind {
    116 ; CHECK-LABEL: test_atomic_load_sub_i16:
    117    %old = atomicrmw sub i16* @var16, i16 %offset release
    118 ; CHECK-NOT: dmb
    119 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    120 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    121 
    122 ; CHECK: .LBB{{[0-9]+}}_1:
    123 ; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    124   ; w0 below is a reasonable guess but could change: it certainly comes into the
    125   ;  function there.
    126 ; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
    127 ; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    128 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    129 ; CHECK-NOT: dmb
    130 
    131 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    132    ret i16 %old
    133 }
    134 
    135 define i32 @test_atomic_load_sub_i32(i32 %offset) nounwind {
    136 ; CHECK-LABEL: test_atomic_load_sub_i32:
    137    %old = atomicrmw sub i32* @var32, i32 %offset acquire
    138 ; CHECK-NOT: dmb
    139 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    140 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    141 
    142 ; CHECK: .LBB{{[0-9]+}}_1:
    143 ; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    144   ; w0 below is a reasonable guess but could change: it certainly comes into the
    145   ;  function there.
    146 ; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
    147 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    148 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    149 ; CHECK-NOT: dmb
    150 
    151 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    152    ret i32 %old
    153 }
    154 
    155 define i64 @test_atomic_load_sub_i64(i64 %offset) nounwind {
    156 ; CHECK-LABEL: test_atomic_load_sub_i64:
    157    %old = atomicrmw sub i64* @var64, i64 %offset seq_cst
    158 ; CHECK-NOT: dmb
    159 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    160 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    161 
    162 ; CHECK: .LBB{{[0-9]+}}_1:
    163 ; ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    164   ; x0 below is a reasonable guess but could change: it certainly comes into the
    165   ; function there.
    166 ; CHECK-NEXT: sub [[NEW:x[0-9]+]], x[[OLD]], x0
    167 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    168 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    169 ; CHECK-NOT: dmb
    170 
    171 ; CHECK: mov x0, x[[OLD]]
    172    ret i64 %old
    173 }
    174 
    175 define i8 @test_atomic_load_and_i8(i8 %offset) nounwind {
    176 ; CHECK-LABEL: test_atomic_load_and_i8:
    177    %old = atomicrmw and i8* @var8, i8 %offset release
    178 ; CHECK-NOT: dmb
    179 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    180 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    181 
    182 ; CHECK: .LBB{{[0-9]+}}_1:
    183 ; ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    184   ; w0 below is a reasonable guess but could change: it certainly comes into the
    185   ;  function there.
    186 ; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
    187 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    188 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    189 ; CHECK-NOT: dmb
    190 
    191 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    192    ret i8 %old
    193 }
    194 
    195 define i16 @test_atomic_load_and_i16(i16 %offset) nounwind {
    196 ; CHECK-LABEL: test_atomic_load_and_i16:
    197    %old = atomicrmw and i16* @var16, i16 %offset monotonic
    198 ; CHECK-NOT: dmb
    199 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    200 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    201 
    202 ; CHECK: .LBB{{[0-9]+}}_1:
    203 ; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    204   ; w0 below is a reasonable guess but could change: it certainly comes into the
    205   ;  function there.
    206 ; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
    207 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    208 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    209 ; CHECK-NOT: dmb
    210 
    211 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    212    ret i16 %old
    213 }
    214 
    215 define i32 @test_atomic_load_and_i32(i32 %offset) nounwind {
    216 ; CHECK-LABEL: test_atomic_load_and_i32:
    217    %old = atomicrmw and i32* @var32, i32 %offset seq_cst
    218 ; CHECK-NOT: dmb
    219 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    220 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    221 
    222 ; CHECK: .LBB{{[0-9]+}}_1:
    223 ; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    224   ; w0 below is a reasonable guess but could change: it certainly comes into the
    225   ;  function there.
    226 ; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
    227 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    228 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    229 ; CHECK-NOT: dmb
    230 
    231 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    232    ret i32 %old
    233 }
    234 
    235 define i64 @test_atomic_load_and_i64(i64 %offset) nounwind {
    236 ; CHECK-LABEL: test_atomic_load_and_i64:
    237    %old = atomicrmw and i64* @var64, i64 %offset acquire
    238 ; CHECK-NOT: dmb
    239 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    240 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    241 
    242 ; CHECK: .LBB{{[0-9]+}}_1:
    243 ; ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    244   ; x0 below is a reasonable guess but could change: it certainly comes into the
    245   ; function there.
    246 ; CHECK-NEXT: and [[NEW:x[0-9]+]], x[[OLD]], x0
    247 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    248 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    249 ; CHECK-NOT: dmb
    250 
    251 ; CHECK: mov x0, x[[OLD]]
    252    ret i64 %old
    253 }
    254 
    255 define i8 @test_atomic_load_or_i8(i8 %offset) nounwind {
    256 ; CHECK-LABEL: test_atomic_load_or_i8:
    257    %old = atomicrmw or i8* @var8, i8 %offset seq_cst
    258 ; CHECK-NOT: dmb
    259 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    260 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    261 
    262 ; CHECK: .LBB{{[0-9]+}}_1:
    263 ; ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    264   ; w0 below is a reasonable guess but could change: it certainly comes into the
    265   ;  function there.
    266 ; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
    267 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    268 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    269 ; CHECK-NOT: dmb
    270 
    271 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    272    ret i8 %old
    273 }
    274 
    275 define i16 @test_atomic_load_or_i16(i16 %offset) nounwind {
    276 ; CHECK-LABEL: test_atomic_load_or_i16:
    277    %old = atomicrmw or i16* @var16, i16 %offset monotonic
    278 ; CHECK-NOT: dmb
    279 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    280 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    281 
    282 ; CHECK: .LBB{{[0-9]+}}_1:
    283 ; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    284   ; w0 below is a reasonable guess but could change: it certainly comes into the
    285   ;  function there.
    286 ; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
    287 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    288 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    289 ; CHECK-NOT: dmb
    290 
    291 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    292    ret i16 %old
    293 }
    294 
    295 define i32 @test_atomic_load_or_i32(i32 %offset) nounwind {
    296 ; CHECK-LABEL: test_atomic_load_or_i32:
    297    %old = atomicrmw or i32* @var32, i32 %offset acquire
    298 ; CHECK-NOT: dmb
    299 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    300 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    301 
    302 ; CHECK: .LBB{{[0-9]+}}_1:
    303 ; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    304   ; w0 below is a reasonable guess but could change: it certainly comes into the
    305   ;  function there.
    306 ; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
    307 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    308 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    309 ; CHECK-NOT: dmb
    310 
    311 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    312    ret i32 %old
    313 }
    314 
    315 define i64 @test_atomic_load_or_i64(i64 %offset) nounwind {
    316 ; CHECK-LABEL: test_atomic_load_or_i64:
    317    %old = atomicrmw or i64* @var64, i64 %offset release
    318 ; CHECK-NOT: dmb
    319 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    320 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    321 
    322 ; CHECK: .LBB{{[0-9]+}}_1:
    323 ; ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    324   ; x0 below is a reasonable guess but could change: it certainly comes into the
    325   ; function there.
    326 ; CHECK-NEXT: orr [[NEW:x[0-9]+]], x[[OLD]], x0
    327 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    328 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    329 ; CHECK-NOT: dmb
    330 
    331 ; CHECK: mov x0, x[[OLD]]
    332    ret i64 %old
    333 }
    334 
    335 define i8 @test_atomic_load_xor_i8(i8 %offset) nounwind {
    336 ; CHECK-LABEL: test_atomic_load_xor_i8:
    337    %old = atomicrmw xor i8* @var8, i8 %offset acquire
    338 ; CHECK-NOT: dmb
    339 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    340 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    341 
    342 ; CHECK: .LBB{{[0-9]+}}_1:
    343 ; ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    344   ; w0 below is a reasonable guess but could change: it certainly comes into the
    345   ;  function there.
    346 ; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
    347 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    348 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    349 ; CHECK-NOT: dmb
    350 
    351 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    352    ret i8 %old
    353 }
    354 
    355 define i16 @test_atomic_load_xor_i16(i16 %offset) nounwind {
    356 ; CHECK-LABEL: test_atomic_load_xor_i16:
    357    %old = atomicrmw xor i16* @var16, i16 %offset release
    358 ; CHECK-NOT: dmb
    359 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    360 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    361 
    362 ; CHECK: .LBB{{[0-9]+}}_1:
    363 ; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    364   ; w0 below is a reasonable guess but could change: it certainly comes into the
    365   ;  function there.
    366 ; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
    367 ; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    368 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    369 ; CHECK-NOT: dmb
    370 
    371 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    372    ret i16 %old
    373 }
    374 
    375 define i32 @test_atomic_load_xor_i32(i32 %offset) nounwind {
    376 ; CHECK-LABEL: test_atomic_load_xor_i32:
    377    %old = atomicrmw xor i32* @var32, i32 %offset seq_cst
    378 ; CHECK-NOT: dmb
    379 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    380 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    381 
    382 ; CHECK: .LBB{{[0-9]+}}_1:
    383 ; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    384   ; w0 below is a reasonable guess but could change: it certainly comes into the
    385   ;  function there.
    386 ; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
    387 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    388 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    389 ; CHECK-NOT: dmb
    390 
    391 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    392    ret i32 %old
    393 }
    394 
    395 define i64 @test_atomic_load_xor_i64(i64 %offset) nounwind {
    396 ; CHECK-LABEL: test_atomic_load_xor_i64:
    397    %old = atomicrmw xor i64* @var64, i64 %offset monotonic
    398 ; CHECK-NOT: dmb
    399 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    400 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    401 
    402 ; CHECK: .LBB{{[0-9]+}}_1:
    403 ; ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    404   ; x0 below is a reasonable guess but could change: it certainly comes into the
    405   ; function there.
    406 ; CHECK-NEXT: eor [[NEW:x[0-9]+]], x[[OLD]], x0
    407 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    408 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    409 ; CHECK-NOT: dmb
    410 
    411 ; CHECK: mov x0, x[[OLD]]
    412    ret i64 %old
    413 }
    414 
    415 define i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
    416 ; CHECK-LABEL: test_atomic_load_xchg_i8:
    417    %old = atomicrmw xchg i8* @var8, i8 %offset monotonic
    418 ; CHECK-NOT: dmb
    419 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    420 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    421 
    422 ; CHECK: .LBB{{[0-9]+}}_1:
    423 ; ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    424   ; w0 below is a reasonable guess but could change: it certainly comes into the
    425   ; function there.
    426 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
    427 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    428 ; CHECK-NOT: dmb
    429 
    430 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    431    ret i8 %old
    432 }
    433 
    434 define i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind {
    435 ; CHECK-LABEL: test_atomic_load_xchg_i16:
    436    %old = atomicrmw xchg i16* @var16, i16 %offset seq_cst
    437 ; CHECK-NOT: dmb
    438 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    439 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    440 
    441 ; CHECK: .LBB{{[0-9]+}}_1:
    442 ; ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    443   ; w0 below is a reasonable guess but could change: it certainly comes into the
    444   ; function there.
    445 ; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
    446 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    447 ; CHECK-NOT: dmb
    448 
    449 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    450    ret i16 %old
    451 }
    452 
    453 define i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind {
    454 ; CHECK-LABEL: test_atomic_load_xchg_i32:
    455 ; CHECK: mov {{[xw]}}8, w[[OLD:[0-9]+]]
    456    %old = atomicrmw xchg i32* @var32, i32 %offset release
    457 ; CHECK-NOT: dmb
    458 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    459 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    460 
    461 ; CHECK: .LBB{{[0-9]+}}_1:
    462 ; ; CHECK: ldxr {{[xw]}}[[OLD]], [x[[ADDR]]]
    463   ; w0 below is a reasonable guess but could change: it certainly comes into the
    464   ;  function there.
    465 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], w8, [x[[ADDR]]]
    466 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    467 ; CHECK-NOT: dmb
    468    ret i32 %old
    469 }
    470 
    471 define i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind {
    472 ; CHECK-LABEL: test_atomic_load_xchg_i64:
    473    %old = atomicrmw xchg i64* @var64, i64 %offset acquire
    474 ; CHECK-NOT: dmb
    475 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    476 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    477 
    478 ; CHECK: .LBB{{[0-9]+}}_1:
    479 ; ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    480   ; x0 below is a reasonable guess but could change: it certainly comes into the
    481   ; function there.
    482 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], x0, [x[[ADDR]]]
    483 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    484 ; CHECK-NOT: dmb
    485 
    486 ; CHECK: mov x0, x[[OLD]]
    487    ret i64 %old
    488 }
    489 
    490 
    491 define i8 @test_atomic_load_min_i8(i8 %offset) nounwind {
    492 ; CHECK-LABEL: test_atomic_load_min_i8:
    493    %old = atomicrmw min i8* @var8, i8 %offset acquire
    494 ; CHECK-NOT: dmb
    495 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    496 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    497 
    498 ; CHECK: .LBB{{[0-9]+}}_1:
    499 ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    500   ; w0 below is a reasonable guess but could change: it certainly comes into the
    501   ;  function there.
    502 
    503 ; CHECK-NEXT: sxtb w[[OLD_EXT:[0-9]+]], w[[OLD]]
    504 ; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxtb
    505 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, le
    506 
    507 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    508 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    509 ; CHECK-NOT: dmb
    510 
    511 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
    512    ret i8 %old
    513 }
    514 
    515 define i16 @test_atomic_load_min_i16(i16 %offset) nounwind {
    516 ; CHECK-LABEL: test_atomic_load_min_i16:
    517    %old = atomicrmw min i16* @var16, i16 %offset release
    518 ; CHECK-NOT: dmb
    519 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    520 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    521 
    522 ; CHECK: .LBB{{[0-9]+}}_1:
    523 ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    524   ; w0 below is a reasonable guess but could change: it certainly comes into the
    525   ;  function there.
    526 
    527 ; CHECK-NEXT: sxth w[[OLD_EXT:[0-9]+]], w[[OLD]]
    528 ; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxth
    529 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, le
    530 
    531 
    532 ; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    533 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    534 ; CHECK-NOT: dmb
    535 
    536 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
    537    ret i16 %old
    538 }
    539 
    540 define i32 @test_atomic_load_min_i32(i32 %offset) nounwind {
    541 ; CHECK-LABEL: test_atomic_load_min_i32:
    542    %old = atomicrmw min i32* @var32, i32 %offset monotonic
    543 ; CHECK-NOT: dmb
    544 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    545 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    546 
    547 ; CHECK: .LBB{{[0-9]+}}_1:
    548 ; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    549   ; w0 below is a reasonable guess but could change: it certainly comes into the
    550   ;  function there.
    551 
    552 ; CHECK-NEXT: cmp w[[OLD]], w0
    553 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, le
    554 
    555 
    556 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    557 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    558 ; CHECK-NOT: dmb
    559 
    560 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    561    ret i32 %old
    562 }
    563 
    564 define i64 @test_atomic_load_min_i64(i64 %offset) nounwind {
    565 ; CHECK-LABEL: test_atomic_load_min_i64:
    566    %old = atomicrmw min i64* @var64, i64 %offset seq_cst
    567 ; CHECK-NOT: dmb
    568 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    569 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    570 
    571 ; CHECK: .LBB{{[0-9]+}}_1:
    572 ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    573   ; x0 below is a reasonable guess but could change: it certainly comes into the
    574   ; function there.
    575 
    576 ; CHECK-NEXT: cmp x[[OLD]], x0
    577 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, le
    578 
    579 
    580 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    581 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    582 ; CHECK-NOT: dmb
    583 
    584 ; CHECK: mov x0, x[[OLD]]
    585    ret i64 %old
    586 }
    587 
    588 define i8 @test_atomic_load_max_i8(i8 %offset) nounwind {
    589 ; CHECK-LABEL: test_atomic_load_max_i8:
    590    %old = atomicrmw max i8* @var8, i8 %offset seq_cst
    591 ; CHECK-NOT: dmb
    592 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    593 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    594 
    595 ; CHECK: .LBB{{[0-9]+}}_1:
    596 ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    597   ; w0 below is a reasonable guess but could change: it certainly comes into the
    598   ;  function there.
    599 
    600 ; CHECK-NEXT: sxtb w[[OLD_EXT:[0-9]+]], w[[OLD]]
    601 ; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxtb
    602 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
    603 
    604 
    605 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    606 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    607 ; CHECK-NOT: dmb
    608 
    609 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
    610    ret i8 %old
    611 }
    612 
    613 define i16 @test_atomic_load_max_i16(i16 %offset) nounwind {
    614 ; CHECK-LABEL: test_atomic_load_max_i16:
    615    %old = atomicrmw max i16* @var16, i16 %offset acquire
    616 ; CHECK-NOT: dmb
    617 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    618 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    619 
    620 ; CHECK: .LBB{{[0-9]+}}_1:
    621 ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    622   ; w0 below is a reasonable guess but could change: it certainly comes into the
    623   ;  function there.
    624 
    625 ; CHECK-NEXT: sxth w[[OLD_EXT:[0-9]+]], w[[OLD]]
    626 ; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxth
    627 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
    628 
    629 
    630 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    631 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    632 ; CHECK-NOT: dmb
    633 
    634 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
    635    ret i16 %old
    636 }
    637 
    638 define i32 @test_atomic_load_max_i32(i32 %offset) nounwind {
    639 ; CHECK-LABEL: test_atomic_load_max_i32:
    640    %old = atomicrmw max i32* @var32, i32 %offset release
    641 ; CHECK-NOT: dmb
    642 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    643 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    644 
    645 ; CHECK: .LBB{{[0-9]+}}_1:
    646 ; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    647   ; w0 below is a reasonable guess but could change: it certainly comes into the
    648   ;  function there.
    649 
    650 ; CHECK-NEXT: cmp w[[OLD]], w0
    651 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
    652 
    653 
    654 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    655 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    656 ; CHECK-NOT: dmb
    657 
    658 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    659    ret i32 %old
    660 }
    661 
    662 define i64 @test_atomic_load_max_i64(i64 %offset) nounwind {
    663 ; CHECK-LABEL: test_atomic_load_max_i64:
    664    %old = atomicrmw max i64* @var64, i64 %offset monotonic
    665 ; CHECK-NOT: dmb
    666 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    667 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    668 
    669 ; CHECK: .LBB{{[0-9]+}}_1:
    670 ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    671   ; x0 below is a reasonable guess but could change: it certainly comes into the
    672   ; function there.
    673 
    674 ; CHECK-NEXT: cmp x[[OLD]], x0
    675 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, gt
    676 
    677 
    678 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    679 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    680 ; CHECK-NOT: dmb
    681 
    682 ; CHECK: mov x0, x[[OLD]]
    683    ret i64 %old
    684 }
    685 
    686 define i8 @test_atomic_load_umin_i8(i8 %offset) nounwind {
    687 ; CHECK-LABEL: test_atomic_load_umin_i8:
    688    %old = atomicrmw umin i8* @var8, i8 %offset monotonic
    689 ; CHECK-NOT: dmb
    690 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    691 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    692 
    693 ; CHECK: .LBB{{[0-9]+}}_1:
    694 ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    695   ; w0 below is a reasonable guess but could change: it certainly comes into the
    696   ;  function there.
    697 
    698 ; CHECK-NEXT: cmp w[[OLD]], w0, uxtb
    699 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, ls
    700 
    701 
    702 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    703 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    704 ; CHECK-NOT: dmb
    705 
    706 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    707    ret i8 %old
    708 }
    709 
    710 define i16 @test_atomic_load_umin_i16(i16 %offset) nounwind {
    711 ; CHECK-LABEL: test_atomic_load_umin_i16:
    712    %old = atomicrmw umin i16* @var16, i16 %offset acquire
    713 ; CHECK-NOT: dmb
    714 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    715 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    716 
    717 ; CHECK: .LBB{{[0-9]+}}_1:
    718 ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    719   ; w0 below is a reasonable guess but could change: it certainly comes into the
    720   ;  function there.
    721 
    722 ; CHECK-NEXT: cmp w[[OLD]], w0, uxth
    723 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, ls
    724 
    725 
    726 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    727 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    728 ; CHECK-NOT: dmb
    729 
    730 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    731    ret i16 %old
    732 }
    733 
    734 define i32 @test_atomic_load_umin_i32(i32 %offset) nounwind {
    735 ; CHECK-LABEL: test_atomic_load_umin_i32:
    736    %old = atomicrmw umin i32* @var32, i32 %offset seq_cst
    737 ; CHECK-NOT: dmb
    738 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    739 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    740 
    741 ; CHECK: .LBB{{[0-9]+}}_1:
    742 ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    743   ; w0 below is a reasonable guess but could change: it certainly comes into the
    744   ;  function there.
    745 
    746 ; CHECK-NEXT: cmp w[[OLD]], w0
    747 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, ls
    748 
    749 
    750 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    751 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    752 ; CHECK-NOT: dmb
    753 
    754 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    755    ret i32 %old
    756 }
    757 
    758 define i64 @test_atomic_load_umin_i64(i64 %offset) nounwind {
    759 ; CHECK-LABEL: test_atomic_load_umin_i64:
    760    %old = atomicrmw umin i64* @var64, i64 %offset acq_rel
    761 ; CHECK-NOT: dmb
    762 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    763 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    764 
    765 ; CHECK: .LBB{{[0-9]+}}_1:
    766 ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    767   ; x0 below is a reasonable guess but could change: it certainly comes into the
    768   ; function there.
    769 
    770 ; CHECK-NEXT: cmp x[[OLD]], x0
    771 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, ls
    772 
    773 
    774 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    775 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    776 ; CHECK-NOT: dmb
    777 
    778 ; CHECK: mov x0, x[[OLD]]
    779    ret i64 %old
    780 }
    781 
    782 define i8 @test_atomic_load_umax_i8(i8 %offset) nounwind {
    783 ; CHECK-LABEL: test_atomic_load_umax_i8:
    784    %old = atomicrmw umax i8* @var8, i8 %offset acq_rel
    785 ; CHECK-NOT: dmb
    786 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    787 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    788 
    789 ; CHECK: .LBB{{[0-9]+}}_1:
    790 ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    791   ; w0 below is a reasonable guess but could change: it certainly comes into the
    792   ;  function there.
    793 
    794 ; CHECK-NEXT: cmp w[[OLD]], w0, uxtb
    795 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
    796 
    797 
    798 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    799 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    800 ; CHECK-NOT: dmb
    801 
    802 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    803    ret i8 %old
    804 }
    805 
    806 define i16 @test_atomic_load_umax_i16(i16 %offset) nounwind {
    807 ; CHECK-LABEL: test_atomic_load_umax_i16:
    808    %old = atomicrmw umax i16* @var16, i16 %offset monotonic
    809 ; CHECK-NOT: dmb
    810 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    811 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    812 
    813 ; CHECK: .LBB{{[0-9]+}}_1:
    814 ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    815   ; w0 below is a reasonable guess but could change: it certainly comes into the
    816   ;  function there.
    817 
    818 ; CHECK-NEXT: cmp w[[OLD]], w0, uxth
    819 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
    820 
    821 
    822 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    823 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    824 ; CHECK-NOT: dmb
    825 
    826 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    827    ret i16 %old
    828 }
    829 
    830 define i32 @test_atomic_load_umax_i32(i32 %offset) nounwind {
    831 ; CHECK-LABEL: test_atomic_load_umax_i32:
    832    %old = atomicrmw umax i32* @var32, i32 %offset seq_cst
    833 ; CHECK-NOT: dmb
    834 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    835 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    836 
    837 ; CHECK: .LBB{{[0-9]+}}_1:
    838 ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    839   ; w0 below is a reasonable guess but could change: it certainly comes into the
    840   ;  function there.
    841 
    842 ; CHECK-NEXT: cmp w[[OLD]], w0
    843 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
    844 
    845 
    846 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    847 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    848 ; CHECK-NOT: dmb
    849 
    850 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    851    ret i32 %old
    852 }
    853 
    854 define i64 @test_atomic_load_umax_i64(i64 %offset) nounwind {
    855 ; CHECK-LABEL: test_atomic_load_umax_i64:
    856    %old = atomicrmw umax i64* @var64, i64 %offset release
    857 ; CHECK-NOT: dmb
    858 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    859 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    860 
    861 ; CHECK: .LBB{{[0-9]+}}_1:
    862 ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    863   ; x0 below is a reasonable guess but could change: it certainly comes into the
    864   ; function there.
    865 
    866 ; CHECK-NEXT: cmp x[[OLD]], x0
    867 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, hi
    868 
    869 
    870 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    871 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    872 ; CHECK-NOT: dmb
    873 
    874 ; CHECK: mov x0, x[[OLD]]
    875    ret i64 %old
    876 }
    877 
    878 define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
    879 ; CHECK-LABEL: test_atomic_cmpxchg_i8:
    880    %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire
    881    %old = extractvalue { i8, i1 } %pair, 0
    882 
    883 ; CHECK-NOT: dmb
    884 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    885 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    886 
    887 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
    888 ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    889   ; w0 below is a reasonable guess but could change: it certainly comes into the
    890   ;  function there.
    891 ; CHECK-NEXT: cmp w[[OLD]], w0
    892 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
    893 ; CHECK: stxrb [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]]
    894 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
    895 ; CHECK: [[GET_OUT]]:
    896 ; CHECK: clrex
    897 ; CHECK-NOT: dmb
    898 
    899 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    900    ret i8 %old
    901 }
    902 
    903 define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
    904 ; CHECK-LABEL: test_atomic_cmpxchg_i16:
    905    %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst
    906    %old = extractvalue { i16, i1 } %pair, 0
    907 
    908 ; CHECK-NOT: dmb
    909 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    910 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    911 
    912 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
    913 ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    914   ; w0 below is a reasonable guess but could change: it certainly comes into the
    915   ;  function there.
    916 ; CHECK-NEXT: cmp w[[OLD]], w0
    917 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
    918 ; CHECK: stlxrh [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]]
    919 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
    920 ; CHECK: [[GET_OUT]]:
    921 ; CHECK: clrex
    922 ; CHECK-NOT: dmb
    923 
    924 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    925    ret i16 %old
    926 }
    927 
    928 define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
    929 ; CHECK-LABEL: test_atomic_cmpxchg_i32:
    930    %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new release monotonic
    931    %old = extractvalue { i32, i1 } %pair, 0
    932 
    933 ; CHECK: mov {{[xw]}}[[WANTED:[0-9]+]], {{[xw]}}0
    934 
    935 ; CHECK-NOT: dmb
    936 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    937 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    938 
    939 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
    940 ; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    941 ; CHECK-NEXT: cmp w[[OLD]], w[[WANTED]]
    942 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
    943 ; CHECK: stlxr [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]]
    944 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
    945 ; CHECK: [[GET_OUT]]:
    946 ; CHECK: clrex
    947 ; CHECK-NOT: dmb
    948    ret i32 %old
    949 }
    950 
    951 define void @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
    952 ; CHECK-LABEL: test_atomic_cmpxchg_i64:
    953    %pair = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic
    954    %old = extractvalue { i64, i1 } %pair, 0
    955 
    956 ; CHECK-NOT: dmb
    957 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    958 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    959 
    960 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
    961 ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    962   ; w0 below is a reasonable guess but could change: it certainly comes into the
    963   ;  function there.
    964 ; CHECK-NEXT: cmp x[[OLD]], x0
    965 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
    966   ; As above, w1 is a reasonable guess.
    967 ; CHECK: stxr [[STATUS:w[0-9]+]], x1, [x[[ADDR]]]
    968 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
    969 ; CHECK: [[GET_OUT]]:
    970 ; CHECK: clrex
    971 ; CHECK-NOT: dmb
    972 
    973 ; CHECK: str x[[OLD]],
    974    store i64 %old, i64* @var64
    975    ret void
    976 }
    977 
    978 define i8 @test_atomic_load_monotonic_i8() nounwind {
    979 ; CHECK-LABEL: test_atomic_load_monotonic_i8:
    980   %val = load atomic i8, i8* @var8 monotonic, align 1
    981 ; CHECK-NOT: dmb
    982 ; CHECK: adrp x[[HIADDR:[0-9]+]], var8
    983 ; CHECK: ldrb w0, [x[[HIADDR]], {{#?}}:lo12:var8]
    984 ; CHECK-NOT: dmb
    985 
    986   ret i8 %val
    987 }
    988 
    989 define i8 @test_atomic_load_monotonic_regoff_i8(i64 %base, i64 %off) nounwind {
    990 ; CHECK-LABEL: test_atomic_load_monotonic_regoff_i8:
    991   %addr_int = add i64 %base, %off
    992   %addr = inttoptr i64 %addr_int to i8*
    993 
    994   %val = load atomic i8, i8* %addr monotonic, align 1
    995 ; CHECK-NOT: dmb
    996 ; CHECK: ldrb w0, [x0, x1]
    997 ; CHECK-NOT: dmb
    998 
    999   ret i8 %val
   1000 }
   1001 
   1002 define i8 @test_atomic_load_acquire_i8() nounwind {
   1003 ; CHECK-LABEL: test_atomic_load_acquire_i8:
   1004   %val = load atomic i8, i8* @var8 acquire, align 1
   1005 ; CHECK-NOT: dmb
   1006 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
   1007 ; CHECK-NOT: dmb
   1008 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
   1009 ; CHECK-NOT: dmb
   1010 ; CHECK: ldarb w0, [x[[ADDR]]]
   1011 ; CHECK-NOT: dmb
   1012   ret i8 %val
   1013 }
   1014 
   1015 define i8 @test_atomic_load_seq_cst_i8() nounwind {
   1016 ; CHECK-LABEL: test_atomic_load_seq_cst_i8:
   1017   %val = load atomic i8, i8* @var8 seq_cst, align 1
   1018 ; CHECK-NOT: dmb
   1019 ; CHECK: adrp [[HIADDR:x[0-9]+]], var8
   1020 ; CHECK-NOT: dmb
   1021 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var8
   1022 ; CHECK-NOT: dmb
   1023 ; CHECK: ldarb w0, [x[[ADDR]]]
   1024 ; CHECK-NOT: dmb
   1025   ret i8 %val
   1026 }
   1027 
   1028 define i16 @test_atomic_load_monotonic_i16() nounwind {
   1029 ; CHECK-LABEL: test_atomic_load_monotonic_i16:
   1030   %val = load atomic i16, i16* @var16 monotonic, align 2
   1031 ; CHECK-NOT: dmb
   1032 ; CHECK: adrp x[[HIADDR:[0-9]+]], var16
   1033 ; CHECK-NOT: dmb
   1034 ; CHECK: ldrh w0, [x[[HIADDR]], {{#?}}:lo12:var16]
   1035 ; CHECK-NOT: dmb
   1036 
   1037   ret i16 %val
   1038 }
   1039 
   1040 define i32 @test_atomic_load_monotonic_regoff_i32(i64 %base, i64 %off) nounwind {
   1041 ; CHECK-LABEL: test_atomic_load_monotonic_regoff_i32:
   1042   %addr_int = add i64 %base, %off
   1043   %addr = inttoptr i64 %addr_int to i32*
   1044 
   1045   %val = load atomic i32, i32* %addr monotonic, align 4
   1046 ; CHECK-NOT: dmb
   1047 ; CHECK: ldr w0, [x0, x1]
   1048 ; CHECK-NOT: dmb
   1049 
   1050   ret i32 %val
   1051 }
   1052 
   1053 define i64 @test_atomic_load_seq_cst_i64() nounwind {
   1054 ; CHECK-LABEL: test_atomic_load_seq_cst_i64:
   1055   %val = load atomic i64, i64* @var64 seq_cst, align 8
   1056 ; CHECK-NOT: dmb
   1057 ; CHECK: adrp [[HIADDR:x[0-9]+]], var64
   1058 ; CHECK-NOT: dmb
   1059 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var64
   1060 ; CHECK-NOT: dmb
   1061 ; CHECK: ldar x0, [x[[ADDR]]]
   1062 ; CHECK-NOT: dmb
   1063   ret i64 %val
   1064 }
   1065 
   1066 define void @test_atomic_store_monotonic_i8(i8 %val) nounwind {
   1067 ; CHECK-LABEL: test_atomic_store_monotonic_i8:
   1068   store atomic i8 %val, i8* @var8 monotonic, align 1
   1069 ; CHECK: adrp x[[HIADDR:[0-9]+]], var8
   1070 ; CHECK: strb w0, [x[[HIADDR]], {{#?}}:lo12:var8]
   1071 
   1072   ret void
   1073 }
   1074 
   1075 define void @test_atomic_store_monotonic_regoff_i8(i64 %base, i64 %off, i8 %val) nounwind {
   1076 ; CHECK-LABEL: test_atomic_store_monotonic_regoff_i8:
   1077 
   1078   %addr_int = add i64 %base, %off
   1079   %addr = inttoptr i64 %addr_int to i8*
   1080 
   1081   store atomic i8 %val, i8* %addr monotonic, align 1
   1082 ; CHECK: strb w2, [x0, x1]
   1083 
   1084   ret void
   1085 }
   1086 define void @test_atomic_store_release_i8(i8 %val) nounwind {
   1087 ; CHECK-LABEL: test_atomic_store_release_i8:
   1088   store atomic i8 %val, i8* @var8 release, align 1
   1089 ; CHECK-NOT: dmb
   1090 ; CHECK: adrp [[HIADDR:x[0-9]+]], var8
   1091 ; CHECK-NOT: dmb
   1092 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var8
   1093 ; CHECK-NOT: dmb
   1094 ; CHECK: stlrb w0, [x[[ADDR]]]
   1095 ; CHECK-NOT: dmb
   1096   ret void
   1097 }
   1098 
   1099 define void @test_atomic_store_seq_cst_i8(i8 %val) nounwind {
   1100 ; CHECK-LABEL: test_atomic_store_seq_cst_i8:
   1101   store atomic i8 %val, i8* @var8 seq_cst, align 1
   1102 ; CHECK-NOT: dmb
   1103 ; CHECK: adrp [[HIADDR:x[0-9]+]], var8
   1104 ; CHECK-NOT: dmb
   1105 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var8
   1106 ; CHECK-NOT: dmb
   1107 ; CHECK: stlrb w0, [x[[ADDR]]]
   1108 ; CHECK-NOT: dmb
   1109 
   1110   ret void
   1111 }
   1112 
   1113 define void @test_atomic_store_monotonic_i16(i16 %val) nounwind {
   1114 ; CHECK-LABEL: test_atomic_store_monotonic_i16:
   1115   store atomic i16 %val, i16* @var16 monotonic, align 2
   1116 ; CHECK-NOT: dmb
   1117 ; CHECK: adrp x[[HIADDR:[0-9]+]], var16
   1118 ; CHECK-NOT: dmb
   1119 ; CHECK: strh w0, [x[[HIADDR]], {{#?}}:lo12:var16]
   1120 ; CHECK-NOT: dmb
   1121   ret void
   1122 }
   1123 
   1124 define void @test_atomic_store_monotonic_regoff_i32(i64 %base, i64 %off, i32 %val) nounwind {
   1125 ; CHECK-LABEL: test_atomic_store_monotonic_regoff_i32:
   1126 
   1127   %addr_int = add i64 %base, %off
   1128   %addr = inttoptr i64 %addr_int to i32*
   1129 
   1130   store atomic i32 %val, i32* %addr monotonic, align 4
   1131 ; CHECK-NOT: dmb
   1132 ; CHECK: str w2, [x0, x1]
   1133 ; CHECK-NOT: dmb
   1134 
   1135   ret void
   1136 }
   1137 
   1138 define void @test_atomic_store_release_i64(i64 %val) nounwind {
   1139 ; CHECK-LABEL: test_atomic_store_release_i64:
   1140   store atomic i64 %val, i64* @var64 release, align 8
   1141 ; CHECK-NOT: dmb
   1142 ; CHECK: adrp [[HIADDR:x[0-9]+]], var64
   1143 ; CHECK-NOT: dmb
   1144 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var64
   1145 ; CHECK-NOT: dmb
   1146 ; CHECK: stlr x0, [x[[ADDR]]]
   1147 ; CHECK-NOT: dmb
   1148   ret void
   1149 }
   1150