Home | History | Annotate | Download | only in AArch64 
      1 ; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK
      2 ; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-REG
      3 
      4 
      5 ; Point of CHECK-REG is to make sure UNPREDICTABLE instructions aren't created
      6 ; (i.e. reusing a register for status & data in store exclusive).
      7 ; CHECK-REG-NOT: stlxrb w[[NEW:[0-9]+]], w[[NEW]], [x{{[0-9]+}}]
      8 ; CHECK-REG-NOT: stlxrb w[[NEW:[0-9]+]], x[[NEW]], [x{{[0-9]+}}]
      9 
     10 @var8 = global i8 0
     11 @var16 = global i16 0
     12 @var32 = global i32 0
     13 @var64 = global i64 0
     14 
     15 define i8 @test_atomic_load_add_i8(i8 %offset) nounwind {
     16 ; CHECK-LABEL: test_atomic_load_add_i8:
     17    %old = atomicrmw add i8* @var8, i8 %offset seq_cst
     18 ; CHECK-NOT: dmb
     19 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
     20 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
     21 
     22 ; CHECK: .LBB{{[0-9]+}}_1:
     23 ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
     24   ; w0 below is a reasonable guess but could change: it certainly comes into the
     25   ;  function there.
     26 ; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
     27 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
     28 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
     29 ; CHECK-NOT: dmb
     30 
     31 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
     32    ret i8 %old
     33 }
     34 
     35 define i16 @test_atomic_load_add_i16(i16 %offset) nounwind {
     36 ; CHECK-LABEL: test_atomic_load_add_i16:
     37    %old = atomicrmw add i16* @var16, i16 %offset acquire
     38 ; CHECK-NOT: dmb
     39 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
     40 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
     41 
     42 ; CHECK: .LBB{{[0-9]+}}_1:
     43 ; ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
     44   ; w0 below is a reasonable guess but could change: it certainly comes into the
     45   ;  function there.
     46 ; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
     47 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
     48 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
     49 ; CHECK-NOT: dmb
     50 
     51 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
     52    ret i16 %old
     53 }
     54 
     55 define i32 @test_atomic_load_add_i32(i32 %offset) nounwind {
     56 ; CHECK-LABEL: test_atomic_load_add_i32:
     57    %old = atomicrmw add i32* @var32, i32 %offset release
     58 ; CHECK-NOT: dmb
     59 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
     60 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
     61 
     62 ; CHECK: .LBB{{[0-9]+}}_1:
     63 ; ; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
     64   ; w0 below is a reasonable guess but could change: it certainly comes into the
     65   ;  function there.
     66 ; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
     67 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
     68 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
     69 ; CHECK-NOT: dmb
     70 
     71 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
     72    ret i32 %old
     73 }
     74 
     75 define i64 @test_atomic_load_add_i64(i64 %offset) nounwind {
     76 ; CHECK-LABEL: test_atomic_load_add_i64:
     77    %old = atomicrmw add i64* @var64, i64 %offset monotonic
     78 ; CHECK-NOT: dmb
     79 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
     80 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
     81 
     82 ; CHECK: .LBB{{[0-9]+}}_1:
     83 ; ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
     84   ; x0 below is a reasonable guess but could change: it certainly comes into the
     85   ; function there.
     86 ; CHECK-NEXT: add [[NEW:x[0-9]+]], x[[OLD]], x0
     87 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
     88 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
     89 ; CHECK-NOT: dmb
     90 
     91 ; CHECK: mov x0, x[[OLD]]
     92    ret i64 %old
     93 }
     94 
     95 define i8 @test_atomic_load_sub_i8(i8 %offset) nounwind {
     96 ; CHECK-LABEL: test_atomic_load_sub_i8:
     97    %old = atomicrmw sub i8* @var8, i8 %offset monotonic
     98 ; CHECK-NOT: dmb
     99 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    100 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    101 
    102 ; CHECK: .LBB{{[0-9]+}}_1:
    103 ; ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    104   ; w0 below is a reasonable guess but could change: it certainly comes into the
    105   ;  function there.
    106 ; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
    107 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    108 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    109 ; CHECK-NOT: dmb
    110 
    111 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    112    ret i8 %old
    113 }
    114 
    115 define i16 @test_atomic_load_sub_i16(i16 %offset) nounwind {
    116 ; CHECK-LABEL: test_atomic_load_sub_i16:
    117    %old = atomicrmw sub i16* @var16, i16 %offset release
    118 ; CHECK-NOT: dmb
    119 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    120 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    121 
    122 ; CHECK: .LBB{{[0-9]+}}_1:
    123 ; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    124   ; w0 below is a reasonable guess but could change: it certainly comes into the
    125   ;  function there.
    126 ; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
    127 ; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    128 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    129 ; CHECK-NOT: dmb
    130 
    131 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    132    ret i16 %old
    133 }
    134 
    135 define i32 @test_atomic_load_sub_i32(i32 %offset) nounwind {
    136 ; CHECK-LABEL: test_atomic_load_sub_i32:
    137    %old = atomicrmw sub i32* @var32, i32 %offset acquire
    138 ; CHECK-NOT: dmb
    139 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    140 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    141 
    142 ; CHECK: .LBB{{[0-9]+}}_1:
    143 ; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    144   ; w0 below is a reasonable guess but could change: it certainly comes into the
    145   ;  function there.
    146 ; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
    147 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    148 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    149 ; CHECK-NOT: dmb
    150 
    151 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    152    ret i32 %old
    153 }
    154 
    155 define i64 @test_atomic_load_sub_i64(i64 %offset) nounwind {
    156 ; CHECK-LABEL: test_atomic_load_sub_i64:
    157    %old = atomicrmw sub i64* @var64, i64 %offset seq_cst
    158 ; CHECK-NOT: dmb
    159 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    160 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    161 
    162 ; CHECK: .LBB{{[0-9]+}}_1:
    163 ; ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    164   ; x0 below is a reasonable guess but could change: it certainly comes into the
    165   ; function there.
    166 ; CHECK-NEXT: sub [[NEW:x[0-9]+]], x[[OLD]], x0
    167 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    168 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    169 ; CHECK-NOT: dmb
    170 
    171 ; CHECK: mov x0, x[[OLD]]
    172    ret i64 %old
    173 }
    174 
    175 define i8 @test_atomic_load_and_i8(i8 %offset) nounwind {
    176 ; CHECK-LABEL: test_atomic_load_and_i8:
    177    %old = atomicrmw and i8* @var8, i8 %offset release
    178 ; CHECK-NOT: dmb
    179 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    180 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    181 
    182 ; CHECK: .LBB{{[0-9]+}}_1:
    183 ; ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    184   ; w0 below is a reasonable guess but could change: it certainly comes into the
    185   ;  function there.
    186 ; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
    187 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    188 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    189 ; CHECK-NOT: dmb
    190 
    191 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    192    ret i8 %old
    193 }
    194 
    195 define i16 @test_atomic_load_and_i16(i16 %offset) nounwind {
    196 ; CHECK-LABEL: test_atomic_load_and_i16:
    197    %old = atomicrmw and i16* @var16, i16 %offset monotonic
    198 ; CHECK-NOT: dmb
    199 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    200 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    201 
    202 ; CHECK: .LBB{{[0-9]+}}_1:
    203 ; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    204   ; w0 below is a reasonable guess but could change: it certainly comes into the
    205   ;  function there.
    206 ; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
    207 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    208 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    209 ; CHECK-NOT: dmb
    210 
    211 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    212    ret i16 %old
    213 }
    214 
    215 define i32 @test_atomic_load_and_i32(i32 %offset) nounwind {
    216 ; CHECK-LABEL: test_atomic_load_and_i32:
    217    %old = atomicrmw and i32* @var32, i32 %offset seq_cst
    218 ; CHECK-NOT: dmb
    219 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    220 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    221 
    222 ; CHECK: .LBB{{[0-9]+}}_1:
    223 ; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    224   ; w0 below is a reasonable guess but could change: it certainly comes into the
    225   ;  function there.
    226 ; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
    227 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    228 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    229 ; CHECK-NOT: dmb
    230 
    231 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    232    ret i32 %old
    233 }
    234 
    235 define i64 @test_atomic_load_and_i64(i64 %offset) nounwind {
    236 ; CHECK-LABEL: test_atomic_load_and_i64:
    237    %old = atomicrmw and i64* @var64, i64 %offset acquire
    238 ; CHECK-NOT: dmb
    239 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    240 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    241 
    242 ; CHECK: .LBB{{[0-9]+}}_1:
    243 ; ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    244   ; x0 below is a reasonable guess but could change: it certainly comes into the
    245   ; function there.
    246 ; CHECK-NEXT: and [[NEW:x[0-9]+]], x[[OLD]], x0
    247 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    248 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    249 ; CHECK-NOT: dmb
    250 
    251 ; CHECK: mov x0, x[[OLD]]
    252    ret i64 %old
    253 }
    254 
    255 define i8 @test_atomic_load_or_i8(i8 %offset) nounwind {
    256 ; CHECK-LABEL: test_atomic_load_or_i8:
    257    %old = atomicrmw or i8* @var8, i8 %offset seq_cst
    258 ; CHECK-NOT: dmb
    259 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    260 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    261 
    262 ; CHECK: .LBB{{[0-9]+}}_1:
    263 ; ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    264   ; w0 below is a reasonable guess but could change: it certainly comes into the
    265   ;  function there.
    266 ; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
    267 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    268 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    269 ; CHECK-NOT: dmb
    270 
    271 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    272    ret i8 %old
    273 }
    274 
    275 define i16 @test_atomic_load_or_i16(i16 %offset) nounwind {
    276 ; CHECK-LABEL: test_atomic_load_or_i16:
    277    %old = atomicrmw or i16* @var16, i16 %offset monotonic
    278 ; CHECK-NOT: dmb
    279 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    280 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    281 
    282 ; CHECK: .LBB{{[0-9]+}}_1:
    283 ; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    284   ; w0 below is a reasonable guess but could change: it certainly comes into the
    285   ;  function there.
    286 ; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
    287 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    288 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    289 ; CHECK-NOT: dmb
    290 
    291 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    292    ret i16 %old
    293 }
    294 
    295 define i32 @test_atomic_load_or_i32(i32 %offset) nounwind {
    296 ; CHECK-LABEL: test_atomic_load_or_i32:
    297    %old = atomicrmw or i32* @var32, i32 %offset acquire
    298 ; CHECK-NOT: dmb
    299 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    300 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    301 
    302 ; CHECK: .LBB{{[0-9]+}}_1:
    303 ; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    304   ; w0 below is a reasonable guess but could change: it certainly comes into the
    305   ;  function there.
    306 ; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
    307 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    308 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    309 ; CHECK-NOT: dmb
    310 
    311 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    312    ret i32 %old
    313 }
    314 
    315 define i64 @test_atomic_load_or_i64(i64 %offset) nounwind {
    316 ; CHECK-LABEL: test_atomic_load_or_i64:
    317    %old = atomicrmw or i64* @var64, i64 %offset release
    318 ; CHECK-NOT: dmb
    319 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    320 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    321 
    322 ; CHECK: .LBB{{[0-9]+}}_1:
    323 ; ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    324   ; x0 below is a reasonable guess but could change: it certainly comes into the
    325   ; function there.
    326 ; CHECK-NEXT: orr [[NEW:x[0-9]+]], x[[OLD]], x0
    327 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    328 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    329 ; CHECK-NOT: dmb
    330 
    331 ; CHECK: mov x0, x[[OLD]]
    332    ret i64 %old
    333 }
    334 
    335 define i8 @test_atomic_load_xor_i8(i8 %offset) nounwind {
    336 ; CHECK-LABEL: test_atomic_load_xor_i8:
    337    %old = atomicrmw xor i8* @var8, i8 %offset acquire
    338 ; CHECK-NOT: dmb
    339 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    340 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    341 
    342 ; CHECK: .LBB{{[0-9]+}}_1:
    343 ; ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    344   ; w0 below is a reasonable guess but could change: it certainly comes into the
    345   ;  function there.
    346 ; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
    347 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    348 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    349 ; CHECK-NOT: dmb
    350 
    351 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    352    ret i8 %old
    353 }
    354 
    355 define i16 @test_atomic_load_xor_i16(i16 %offset) nounwind {
    356 ; CHECK-LABEL: test_atomic_load_xor_i16:
    357    %old = atomicrmw xor i16* @var16, i16 %offset release
    358 ; CHECK-NOT: dmb
    359 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    360 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    361 
    362 ; CHECK: .LBB{{[0-9]+}}_1:
    363 ; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    364   ; w0 below is a reasonable guess but could change: it certainly comes into the
    365   ;  function there.
    366 ; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
    367 ; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    368 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    369 ; CHECK-NOT: dmb
    370 
    371 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    372    ret i16 %old
    373 }
    374 
    375 define i32 @test_atomic_load_xor_i32(i32 %offset) nounwind {
    376 ; CHECK-LABEL: test_atomic_load_xor_i32:
    377    %old = atomicrmw xor i32* @var32, i32 %offset seq_cst
    378 ; CHECK-NOT: dmb
    379 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    380 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    381 
    382 ; CHECK: .LBB{{[0-9]+}}_1:
    383 ; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    384   ; w0 below is a reasonable guess but could change: it certainly comes into the
    385   ;  function there.
    386 ; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
    387 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    388 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    389 ; CHECK-NOT: dmb
    390 
    391 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    392    ret i32 %old
    393 }
    394 
    395 define i64 @test_atomic_load_xor_i64(i64 %offset) nounwind {
    396 ; CHECK-LABEL: test_atomic_load_xor_i64:
    397    %old = atomicrmw xor i64* @var64, i64 %offset monotonic
    398 ; CHECK-NOT: dmb
    399 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    400 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    401 
    402 ; CHECK: .LBB{{[0-9]+}}_1:
    403 ; ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    404   ; x0 below is a reasonable guess but could change: it certainly comes into the
    405   ; function there.
    406 ; CHECK-NEXT: eor [[NEW:x[0-9]+]], x[[OLD]], x0
    407 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    408 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    409 ; CHECK-NOT: dmb
    410 
    411 ; CHECK: mov x0, x[[OLD]]
    412    ret i64 %old
    413 }
    414 
    415 define i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
    416 ; CHECK-LABEL: test_atomic_load_xchg_i8:
    417    %old = atomicrmw xchg i8* @var8, i8 %offset monotonic
    418 ; CHECK-NOT: dmb
    419 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    420 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    421 
    422 ; CHECK: .LBB{{[0-9]+}}_1:
    423 ; ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    424   ; w0 below is a reasonable guess but could change: it certainly comes into the
    425   ; function there.
    426 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
    427 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    428 ; CHECK-NOT: dmb
    429 
    430 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    431    ret i8 %old
    432 }
    433 
    434 define i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind {
    435 ; CHECK-LABEL: test_atomic_load_xchg_i16:
    436    %old = atomicrmw xchg i16* @var16, i16 %offset seq_cst
    437 ; CHECK-NOT: dmb
    438 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    439 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    440 
    441 ; CHECK: .LBB{{[0-9]+}}_1:
    442 ; ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    443   ; w0 below is a reasonable guess but could change: it certainly comes into the
    444   ; function there.
    445 ; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
    446 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    447 ; CHECK-NOT: dmb
    448 
    449 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    450    ret i16 %old
    451 }
    452 
    453 define i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind {
    454 ; CHECK-LABEL: test_atomic_load_xchg_i32:
    455    %old = atomicrmw xchg i32* @var32, i32 %offset release
    456 ; CHECK-NOT: dmb
    457 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    458 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    459 
    460 ; CHECK: .LBB{{[0-9]+}}_1:
    461 ; ; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    462   ; w0 below is a reasonable guess but could change: it certainly comes into the
    463   ;  function there.
    464 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
    465 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    466 ; CHECK-NOT: dmb
    467 
    468 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    469    ret i32 %old
    470 }
    471 
    472 define i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind {
    473 ; CHECK-LABEL: test_atomic_load_xchg_i64:
    474    %old = atomicrmw xchg i64* @var64, i64 %offset acquire
    475 ; CHECK-NOT: dmb
    476 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    477 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    478 
    479 ; CHECK: .LBB{{[0-9]+}}_1:
    480 ; ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    481   ; x0 below is a reasonable guess but could change: it certainly comes into the
    482   ; function there.
    483 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], x0, [x[[ADDR]]]
    484 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    485 ; CHECK-NOT: dmb
    486 
    487 ; CHECK: mov x0, x[[OLD]]
    488    ret i64 %old
    489 }
    490 
    491 
    492 define i8 @test_atomic_load_min_i8(i8 %offset) nounwind {
    493 ; CHECK-LABEL: test_atomic_load_min_i8:
    494    %old = atomicrmw min i8* @var8, i8 %offset acquire
    495 ; CHECK-NOT: dmb
    496 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    497 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    498 
    499 ; CHECK: .LBB{{[0-9]+}}_1:
    500 ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    501   ; w0 below is a reasonable guess but could change: it certainly comes into the
    502   ;  function there.
    503 
    504 ; CHECK-NEXT: sxtb w[[OLD_EXT:[0-9]+]], w[[OLD]]
    505 ; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxtb
    506 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, le
    507 
    508 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    509 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    510 ; CHECK-NOT: dmb
    511 
    512 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
    513    ret i8 %old
    514 }
    515 
    516 define i16 @test_atomic_load_min_i16(i16 %offset) nounwind {
    517 ; CHECK-LABEL: test_atomic_load_min_i16:
    518    %old = atomicrmw min i16* @var16, i16 %offset release
    519 ; CHECK-NOT: dmb
    520 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    521 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    522 
    523 ; CHECK: .LBB{{[0-9]+}}_1:
    524 ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    525   ; w0 below is a reasonable guess but could change: it certainly comes into the
    526   ;  function there.
    527 
    528 ; CHECK-NEXT: sxth w[[OLD_EXT:[0-9]+]], w[[OLD]]
    529 ; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxth
    530 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, le
    531 
    532 
    533 ; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    534 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    535 ; CHECK-NOT: dmb
    536 
    537 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
    538    ret i16 %old
    539 }
    540 
    541 define i32 @test_atomic_load_min_i32(i32 %offset) nounwind {
    542 ; CHECK-LABEL: test_atomic_load_min_i32:
    543    %old = atomicrmw min i32* @var32, i32 %offset monotonic
    544 ; CHECK-NOT: dmb
    545 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    546 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    547 
    548 ; CHECK: .LBB{{[0-9]+}}_1:
    549 ; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    550   ; w0 below is a reasonable guess but could change: it certainly comes into the
    551   ;  function there.
    552 
    553 ; CHECK-NEXT: cmp w[[OLD]], w0
    554 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, le
    555 
    556 
    557 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    558 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    559 ; CHECK-NOT: dmb
    560 
    561 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    562    ret i32 %old
    563 }
    564 
    565 define i64 @test_atomic_load_min_i64(i64 %offset) nounwind {
    566 ; CHECK-LABEL: test_atomic_load_min_i64:
    567    %old = atomicrmw min i64* @var64, i64 %offset seq_cst
    568 ; CHECK-NOT: dmb
    569 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    570 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    571 
    572 ; CHECK: .LBB{{[0-9]+}}_1:
    573 ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    574   ; x0 below is a reasonable guess but could change: it certainly comes into the
    575   ; function there.
    576 
    577 ; CHECK-NEXT: cmp x[[OLD]], x0
    578 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, le
    579 
    580 
    581 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    582 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    583 ; CHECK-NOT: dmb
    584 
    585 ; CHECK: mov x0, x[[OLD]]
    586    ret i64 %old
    587 }
    588 
    589 define i8 @test_atomic_load_max_i8(i8 %offset) nounwind {
    590 ; CHECK-LABEL: test_atomic_load_max_i8:
    591    %old = atomicrmw max i8* @var8, i8 %offset seq_cst
    592 ; CHECK-NOT: dmb
    593 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    594 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    595 
    596 ; CHECK: .LBB{{[0-9]+}}_1:
    597 ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    598   ; w0 below is a reasonable guess but could change: it certainly comes into the
    599   ;  function there.
    600 
    601 ; CHECK-NEXT: sxtb w[[OLD_EXT:[0-9]+]], w[[OLD]]
    602 ; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxtb
    603 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
    604 
    605 
    606 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    607 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    608 ; CHECK-NOT: dmb
    609 
    610 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
    611    ret i8 %old
    612 }
    613 
    614 define i16 @test_atomic_load_max_i16(i16 %offset) nounwind {
    615 ; CHECK-LABEL: test_atomic_load_max_i16:
    616    %old = atomicrmw max i16* @var16, i16 %offset acquire
    617 ; CHECK-NOT: dmb
    618 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    619 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    620 
    621 ; CHECK: .LBB{{[0-9]+}}_1:
    622 ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    623   ; w0 below is a reasonable guess but could change: it certainly comes into the
    624   ;  function there.
    625 
    626 ; CHECK-NEXT: sxth w[[OLD_EXT:[0-9]+]], w[[OLD]]
    627 ; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxth
    628 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
    629 
    630 
    631 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    632 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    633 ; CHECK-NOT: dmb
    634 
    635 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
    636    ret i16 %old
    637 }
    638 
    639 define i32 @test_atomic_load_max_i32(i32 %offset) nounwind {
    640 ; CHECK-LABEL: test_atomic_load_max_i32:
    641    %old = atomicrmw max i32* @var32, i32 %offset release
    642 ; CHECK-NOT: dmb
    643 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    644 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    645 
    646 ; CHECK: .LBB{{[0-9]+}}_1:
    647 ; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    648   ; w0 below is a reasonable guess but could change: it certainly comes into the
    649   ;  function there.
    650 
    651 ; CHECK-NEXT: cmp w[[OLD]], w0
    652 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
    653 
    654 
    655 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    656 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    657 ; CHECK-NOT: dmb
    658 
    659 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    660    ret i32 %old
    661 }
    662 
    663 define i64 @test_atomic_load_max_i64(i64 %offset) nounwind {
    664 ; CHECK-LABEL: test_atomic_load_max_i64:
    665    %old = atomicrmw max i64* @var64, i64 %offset monotonic
    666 ; CHECK-NOT: dmb
    667 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    668 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    669 
    670 ; CHECK: .LBB{{[0-9]+}}_1:
    671 ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    672   ; x0 below is a reasonable guess but could change: it certainly comes into the
    673   ; function there.
    674 
    675 ; CHECK-NEXT: cmp x[[OLD]], x0
    676 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, gt
    677 
    678 
    679 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    680 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    681 ; CHECK-NOT: dmb
    682 
    683 ; CHECK: mov x0, x[[OLD]]
    684    ret i64 %old
    685 }
    686 
    687 define i8 @test_atomic_load_umin_i8(i8 %offset) nounwind {
    688 ; CHECK-LABEL: test_atomic_load_umin_i8:
    689    %old = atomicrmw umin i8* @var8, i8 %offset monotonic
    690 ; CHECK-NOT: dmb
    691 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    692 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    693 
    694 ; CHECK: .LBB{{[0-9]+}}_1:
    695 ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    696   ; w0 below is a reasonable guess but could change: it certainly comes into the
    697   ;  function there.
    698 
    699 ; CHECK-NEXT: cmp w[[OLD]], w0, uxtb
    700 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, ls
    701 
    702 
    703 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    704 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    705 ; CHECK-NOT: dmb
    706 
    707 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    708    ret i8 %old
    709 }
    710 
    711 define i16 @test_atomic_load_umin_i16(i16 %offset) nounwind {
    712 ; CHECK-LABEL: test_atomic_load_umin_i16:
    713    %old = atomicrmw umin i16* @var16, i16 %offset acquire
    714 ; CHECK-NOT: dmb
    715 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    716 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    717 
    718 ; CHECK: .LBB{{[0-9]+}}_1:
    719 ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    720   ; w0 below is a reasonable guess but could change: it certainly comes into the
    721   ;  function there.
    722 
    723 ; CHECK-NEXT: cmp w[[OLD]], w0, uxth
    724 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, ls
    725 
    726 
    727 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    728 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    729 ; CHECK-NOT: dmb
    730 
    731 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    732    ret i16 %old
    733 }
    734 
    735 define i32 @test_atomic_load_umin_i32(i32 %offset) nounwind {
    736 ; CHECK-LABEL: test_atomic_load_umin_i32:
    737    %old = atomicrmw umin i32* @var32, i32 %offset seq_cst
    738 ; CHECK-NOT: dmb
    739 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    740 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    741 
    742 ; CHECK: .LBB{{[0-9]+}}_1:
    743 ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    744   ; w0 below is a reasonable guess but could change: it certainly comes into the
    745   ;  function there.
    746 
    747 ; CHECK-NEXT: cmp w[[OLD]], w0
    748 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, ls
    749 
    750 
    751 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    752 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    753 ; CHECK-NOT: dmb
    754 
    755 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    756    ret i32 %old
    757 }
    758 
    759 define i64 @test_atomic_load_umin_i64(i64 %offset) nounwind {
    760 ; CHECK-LABEL: test_atomic_load_umin_i64:
    761    %old = atomicrmw umin i64* @var64, i64 %offset acq_rel
    762 ; CHECK-NOT: dmb
    763 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    764 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    765 
    766 ; CHECK: .LBB{{[0-9]+}}_1:
    767 ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    768   ; x0 below is a reasonable guess but could change: it certainly comes into the
    769   ; function there.
    770 
    771 ; CHECK-NEXT: cmp x[[OLD]], x0
    772 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, ls
    773 
    774 
    775 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    776 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    777 ; CHECK-NOT: dmb
    778 
    779 ; CHECK: mov x0, x[[OLD]]
    780    ret i64 %old
    781 }
    782 
    783 define i8 @test_atomic_load_umax_i8(i8 %offset) nounwind {
    784 ; CHECK-LABEL: test_atomic_load_umax_i8:
    785    %old = atomicrmw umax i8* @var8, i8 %offset acq_rel
    786 ; CHECK-NOT: dmb
    787 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    788 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    789 
    790 ; CHECK: .LBB{{[0-9]+}}_1:
    791 ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    792   ; w0 below is a reasonable guess but could change: it certainly comes into the
    793   ;  function there.
    794 
    795 ; CHECK-NEXT: cmp w[[OLD]], w0, uxtb
    796 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
    797 
    798 
    799 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    800 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    801 ; CHECK-NOT: dmb
    802 
    803 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    804    ret i8 %old
    805 }
    806 
    807 define i16 @test_atomic_load_umax_i16(i16 %offset) nounwind {
    808 ; CHECK-LABEL: test_atomic_load_umax_i16:
    809    %old = atomicrmw umax i16* @var16, i16 %offset monotonic
    810 ; CHECK-NOT: dmb
    811 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    812 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    813 
    814 ; CHECK: .LBB{{[0-9]+}}_1:
    815 ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    816   ; w0 below is a reasonable guess but could change: it certainly comes into the
    817   ;  function there.
    818 
    819 ; CHECK-NEXT: cmp w[[OLD]], w0, uxth
    820 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
    821 
    822 
    823 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    824 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    825 ; CHECK-NOT: dmb
    826 
    827 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    828    ret i16 %old
    829 }
    830 
    831 define i32 @test_atomic_load_umax_i32(i32 %offset) nounwind {
    832 ; CHECK-LABEL: test_atomic_load_umax_i32:
    833    %old = atomicrmw umax i32* @var32, i32 %offset seq_cst
    834 ; CHECK-NOT: dmb
    835 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    836 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    837 
    838 ; CHECK: .LBB{{[0-9]+}}_1:
    839 ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    840   ; w0 below is a reasonable guess but could change: it certainly comes into the
    841   ;  function there.
    842 
    843 ; CHECK-NEXT: cmp w[[OLD]], w0
    844 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
    845 
    846 
    847 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    848 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    849 ; CHECK-NOT: dmb
    850 
    851 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    852    ret i32 %old
    853 }
    854 
    855 define i64 @test_atomic_load_umax_i64(i64 %offset) nounwind {
    856 ; CHECK-LABEL: test_atomic_load_umax_i64:
    857    %old = atomicrmw umax i64* @var64, i64 %offset release
    858 ; CHECK-NOT: dmb
    859 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    860 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    861 
    862 ; CHECK: .LBB{{[0-9]+}}_1:
    863 ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    864   ; x0 below is a reasonable guess but could change: it certainly comes into the
    865   ; function there.
    866 
    867 ; CHECK-NEXT: cmp x[[OLD]], x0
    868 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, hi
    869 
    870 
    871 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    872 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    873 ; CHECK-NOT: dmb
    874 
    875 ; CHECK: mov x0, x[[OLD]]
    876    ret i64 %old
    877 }
    878 
    879 define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
    880 ; CHECK-LABEL: test_atomic_cmpxchg_i8:
    881    %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire
    882    %old = extractvalue { i8, i1 } %pair, 0
    883 
    884 ; CHECK-NOT: dmb
    885 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    886 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
    887 
    888 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
    889 ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    890   ; w0 below is a reasonable guess but could change: it certainly comes into the
    891   ;  function there.
    892 ; CHECK-NEXT: cmp w[[OLD]], w0
    893 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
    894 ; CHECK: stxrb [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]]
    895 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
    896 ; CHECK: [[GET_OUT]]:
    897 ; CHECK: clrex
    898 ; CHECK-NOT: dmb
    899 
    900 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    901    ret i8 %old
    902 }
    903 
    904 define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
    905 ; CHECK-LABEL: test_atomic_cmpxchg_i16:
    906    %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst
    907    %old = extractvalue { i16, i1 } %pair, 0
    908 
    909 ; CHECK-NOT: dmb
    910 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    911 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
    912 
    913 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
    914 ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    915   ; w0 below is a reasonable guess but could change: it certainly comes into the
    916   ;  function there.
    917 ; CHECK-NEXT: cmp w[[OLD]], w0
    918 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
    919 ; CHECK: stlxrh [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]]
    920 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
    921 ; CHECK: [[GET_OUT]]:
    922 ; CHECK: clrex
    923 ; CHECK-NOT: dmb
    924 
    925 ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
    926    ret i16 %old
    927 }
    928 
    929 define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
    930 ; CHECK-LABEL: test_atomic_cmpxchg_i32:
    931    %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new release monotonic
    932    %old = extractvalue { i32, i1 } %pair, 0
    933 
    934 ; CHECK: mov {{[xw]}}[[WANTED:[0-9]+]], {{[xw]}}0
    935 
    936 ; CHECK-NOT: dmb
    937 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    938 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
    939 
    940 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
    941 ; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    942 ; CHECK-NEXT: cmp w[[OLD]], w[[WANTED]]
    943 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
    944 ; CHECK: stlxr [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]]
    945 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
    946 ; CHECK: [[GET_OUT]]:
    947 ; CHECK: clrex
    948 ; CHECK-NOT: dmb
    949    ret i32 %old
    950 }
    951 
    952 define void @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
    953 ; CHECK-LABEL: test_atomic_cmpxchg_i64:
    954    %pair = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic
    955    %old = extractvalue { i64, i1 } %pair, 0
    956 
    957 ; CHECK-NOT: dmb
    958 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    959 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
    960 
    961 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
    962 ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    963   ; w0 below is a reasonable guess but could change: it certainly comes into the
    964   ;  function there.
    965 ; CHECK-NEXT: cmp x[[OLD]], x0
    966 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
    967   ; As above, w1 is a reasonable guess.
    968 ; CHECK: stxr [[STATUS:w[0-9]+]], x1, [x[[ADDR]]]
    969 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
    970 ; CHECK: [[GET_OUT]]:
    971 ; CHECK: clrex
    972 ; CHECK-NOT: dmb
    973 
    974 ; CHECK: str x[[OLD]],
    975    store i64 %old, i64* @var64
    976    ret void
    977 }
    978 
    979 define i8 @test_atomic_load_monotonic_i8() nounwind {
    980 ; CHECK-LABEL: test_atomic_load_monotonic_i8:
    981   %val = load atomic i8, i8* @var8 monotonic, align 1
    982 ; CHECK-NOT: dmb
    983 ; CHECK: adrp x[[HIADDR:[0-9]+]], var8
    984 ; CHECK: ldrb w0, [x[[HIADDR]], {{#?}}:lo12:var8]
    985 ; CHECK-NOT: dmb
    986 
    987   ret i8 %val
    988 }
    989 
    990 define i8 @test_atomic_load_monotonic_regoff_i8(i64 %base, i64 %off) nounwind {
    991 ; CHECK-LABEL: test_atomic_load_monotonic_regoff_i8:
    992   %addr_int = add i64 %base, %off
    993   %addr = inttoptr i64 %addr_int to i8*
    994 
    995   %val = load atomic i8, i8* %addr monotonic, align 1
    996 ; CHECK-NOT: dmb
    997 ; CHECK: ldrb w0, [x0, x1]
    998 ; CHECK-NOT: dmb
    999 
   1000   ret i8 %val
   1001 }
   1002 
   1003 define i8 @test_atomic_load_acquire_i8() nounwind {
   1004 ; CHECK-LABEL: test_atomic_load_acquire_i8:
   1005   %val = load atomic i8, i8* @var8 acquire, align 1
   1006 ; CHECK-NOT: dmb
   1007 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
   1008 ; CHECK-NOT: dmb
   1009 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
   1010 ; CHECK-NOT: dmb
   1011 ; CHECK: ldarb w0, [x[[ADDR]]]
   1012 ; CHECK-NOT: dmb
   1013   ret i8 %val
   1014 }
   1015 
   1016 define i8 @test_atomic_load_seq_cst_i8() nounwind {
   1017 ; CHECK-LABEL: test_atomic_load_seq_cst_i8:
   1018   %val = load atomic i8, i8* @var8 seq_cst, align 1
   1019 ; CHECK-NOT: dmb
   1020 ; CHECK: adrp [[HIADDR:x[0-9]+]], var8
   1021 ; CHECK-NOT: dmb
   1022 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var8
   1023 ; CHECK-NOT: dmb
   1024 ; CHECK: ldarb w0, [x[[ADDR]]]
   1025 ; CHECK-NOT: dmb
   1026   ret i8 %val
   1027 }
   1028 
   1029 define i16 @test_atomic_load_monotonic_i16() nounwind {
   1030 ; CHECK-LABEL: test_atomic_load_monotonic_i16:
   1031   %val = load atomic i16, i16* @var16 monotonic, align 2
   1032 ; CHECK-NOT: dmb
   1033 ; CHECK: adrp x[[HIADDR:[0-9]+]], var16
   1034 ; CHECK-NOT: dmb
   1035 ; CHECK: ldrh w0, [x[[HIADDR]], {{#?}}:lo12:var16]
   1036 ; CHECK-NOT: dmb
   1037 
   1038   ret i16 %val
   1039 }
   1040 
   1041 define i32 @test_atomic_load_monotonic_regoff_i32(i64 %base, i64 %off) nounwind {
   1042 ; CHECK-LABEL: test_atomic_load_monotonic_regoff_i32:
   1043   %addr_int = add i64 %base, %off
   1044   %addr = inttoptr i64 %addr_int to i32*
   1045 
   1046   %val = load atomic i32, i32* %addr monotonic, align 4
   1047 ; CHECK-NOT: dmb
   1048 ; CHECK: ldr w0, [x0, x1]
   1049 ; CHECK-NOT: dmb
   1050 
   1051   ret i32 %val
   1052 }
   1053 
   1054 define i64 @test_atomic_load_seq_cst_i64() nounwind {
   1055 ; CHECK-LABEL: test_atomic_load_seq_cst_i64:
   1056   %val = load atomic i64, i64* @var64 seq_cst, align 8
   1057 ; CHECK-NOT: dmb
   1058 ; CHECK: adrp [[HIADDR:x[0-9]+]], var64
   1059 ; CHECK-NOT: dmb
   1060 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var64
   1061 ; CHECK-NOT: dmb
   1062 ; CHECK: ldar x0, [x[[ADDR]]]
   1063 ; CHECK-NOT: dmb
   1064   ret i64 %val
   1065 }
   1066 
   1067 define void @test_atomic_store_monotonic_i8(i8 %val) nounwind {
   1068 ; CHECK-LABEL: test_atomic_store_monotonic_i8:
   1069   store atomic i8 %val, i8* @var8 monotonic, align 1
   1070 ; CHECK: adrp x[[HIADDR:[0-9]+]], var8
   1071 ; CHECK: strb w0, [x[[HIADDR]], {{#?}}:lo12:var8]
   1072 
   1073   ret void
   1074 }
   1075 
   1076 define void @test_atomic_store_monotonic_regoff_i8(i64 %base, i64 %off, i8 %val) nounwind {
   1077 ; CHECK-LABEL: test_atomic_store_monotonic_regoff_i8:
   1078 
   1079   %addr_int = add i64 %base, %off
   1080   %addr = inttoptr i64 %addr_int to i8*
   1081 
   1082   store atomic i8 %val, i8* %addr monotonic, align 1
   1083 ; CHECK: strb w2, [x0, x1]
   1084 
   1085   ret void
   1086 }
   1087 define void @test_atomic_store_release_i8(i8 %val) nounwind {
   1088 ; CHECK-LABEL: test_atomic_store_release_i8:
   1089   store atomic i8 %val, i8* @var8 release, align 1
   1090 ; CHECK-NOT: dmb
   1091 ; CHECK: adrp [[HIADDR:x[0-9]+]], var8
   1092 ; CHECK-NOT: dmb
   1093 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var8
   1094 ; CHECK-NOT: dmb
   1095 ; CHECK: stlrb w0, [x[[ADDR]]]
   1096 ; CHECK-NOT: dmb
   1097   ret void
   1098 }
   1099 
   1100 define void @test_atomic_store_seq_cst_i8(i8 %val) nounwind {
   1101 ; CHECK-LABEL: test_atomic_store_seq_cst_i8:
   1102   store atomic i8 %val, i8* @var8 seq_cst, align 1
   1103 ; CHECK-NOT: dmb
   1104 ; CHECK: adrp [[HIADDR:x[0-9]+]], var8
   1105 ; CHECK-NOT: dmb
   1106 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var8
   1107 ; CHECK-NOT: dmb
   1108 ; CHECK: stlrb w0, [x[[ADDR]]]
   1109 ; CHECK-NOT: dmb
   1110 
   1111   ret void
   1112 }
   1113 
   1114 define void @test_atomic_store_monotonic_i16(i16 %val) nounwind {
   1115 ; CHECK-LABEL: test_atomic_store_monotonic_i16:
   1116   store atomic i16 %val, i16* @var16 monotonic, align 2
   1117 ; CHECK-NOT: dmb
   1118 ; CHECK: adrp x[[HIADDR:[0-9]+]], var16
   1119 ; CHECK-NOT: dmb
   1120 ; CHECK: strh w0, [x[[HIADDR]], {{#?}}:lo12:var16]
   1121 ; CHECK-NOT: dmb
   1122   ret void
   1123 }
   1124 
   1125 define void @test_atomic_store_monotonic_regoff_i32(i64 %base, i64 %off, i32 %val) nounwind {
   1126 ; CHECK-LABEL: test_atomic_store_monotonic_regoff_i32:
   1127 
   1128   %addr_int = add i64 %base, %off
   1129   %addr = inttoptr i64 %addr_int to i32*
   1130 
   1131   store atomic i32 %val, i32* %addr monotonic, align 4
   1132 ; CHECK-NOT: dmb
   1133 ; CHECK: str w2, [x0, x1]
   1134 ; CHECK-NOT: dmb
   1135 
   1136   ret void
   1137 }
   1138 
   1139 define void @test_atomic_store_release_i64(i64 %val) nounwind {
   1140 ; CHECK-LABEL: test_atomic_store_release_i64:
   1141   store atomic i64 %val, i64* @var64 release, align 8
   1142 ; CHECK-NOT: dmb
   1143 ; CHECK: adrp [[HIADDR:x[0-9]+]], var64
   1144 ; CHECK-NOT: dmb
   1145 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var64
   1146 ; CHECK-NOT: dmb
   1147 ; CHECK: stlr x0, [x[[ADDR]]]
   1148 ; CHECK-NOT: dmb
   1149   ret void
   1150 }
   1151