Home | History | Annotate | Download | only in AArch64 
      1 ; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s
      2 
      3 @var8 = global i8 0
      4 @var16 = global i16 0
      5 @var32 = global i32 0
      6 @var64 = global i64 0
      7 
      8 define i8 @test_atomic_load_add_i8(i8 %offset) nounwind {
      9 ; CHECK-LABEL: test_atomic_load_add_i8:
     10    %old = atomicrmw add i8* @var8, i8 %offset seq_cst
     11 ; CHECK-NOT: dmb
     12 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
     13 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
     14 
     15 ; CHECK: .LBB{{[0-9]+}}_1:
     16 ; CHECK-NEXT: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
     17   ; w0 below is a reasonable guess but could change: it certainly comes into the
     18   ;  function there.
     19 ; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
     20 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
     21 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
     22 ; CHECK-NOT: dmb
     23 
     24 ; CHECK: mov x0, x[[OLD]]
     25    ret i8 %old
     26 }
     27 
     28 define i16 @test_atomic_load_add_i16(i16 %offset) nounwind {
     29 ; CHECK-LABEL: test_atomic_load_add_i16:
     30    %old = atomicrmw add i16* @var16, i16 %offset acquire
     31 ; CHECK-NOT: dmb
     32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
     33 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
     34 
     35 ; CHECK: .LBB{{[0-9]+}}_1:
     36 ; CHECK-NEXT: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
     37   ; w0 below is a reasonable guess but could change: it certainly comes into the
     38   ;  function there.
     39 ; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
     40 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
     41 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
     42 ; CHECK-NOT: dmb
     43 
     44 ; CHECK: mov x0, x[[OLD]]
     45    ret i16 %old
     46 }
     47 
     48 define i32 @test_atomic_load_add_i32(i32 %offset) nounwind {
     49 ; CHECK-LABEL: test_atomic_load_add_i32:
     50    %old = atomicrmw add i32* @var32, i32 %offset release
     51 ; CHECK-NOT: dmb
     52 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
     53 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
     54 
     55 ; CHECK: .LBB{{[0-9]+}}_1:
     56 ; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
     57   ; w0 below is a reasonable guess but could change: it certainly comes into the
     58   ;  function there.
     59 ; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
     60 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
     61 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
     62 ; CHECK-NOT: dmb
     63 
     64 ; CHECK: mov x0, x[[OLD]]
     65    ret i32 %old
     66 }
     67 
     68 define i64 @test_atomic_load_add_i64(i64 %offset) nounwind {
     69 ; CHECK-LABEL: test_atomic_load_add_i64:
     70    %old = atomicrmw add i64* @var64, i64 %offset monotonic
     71 ; CHECK-NOT: dmb
     72 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
     73 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
     74 
     75 ; CHECK: .LBB{{[0-9]+}}_1:
     76 ; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
     77   ; x0 below is a reasonable guess but could change: it certainly comes into the
     78   ; function there.
     79 ; CHECK-NEXT: add [[NEW:x[0-9]+]], x[[OLD]], x0
     80 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
     81 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
     82 ; CHECK-NOT: dmb
     83 
     84 ; CHECK: mov x0, x[[OLD]]
     85    ret i64 %old
     86 }
     87 
     88 define i8 @test_atomic_load_sub_i8(i8 %offset) nounwind {
     89 ; CHECK-LABEL: test_atomic_load_sub_i8:
     90    %old = atomicrmw sub i8* @var8, i8 %offset monotonic
     91 ; CHECK-NOT: dmb
     92 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
     93 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
     94 
     95 ; CHECK: .LBB{{[0-9]+}}_1:
     96 ; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
     97   ; w0 below is a reasonable guess but could change: it certainly comes into the
     98   ;  function there.
     99 ; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
    100 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    101 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    102 ; CHECK-NOT: dmb
    103 
    104 ; CHECK: mov x0, x[[OLD]]
    105    ret i8 %old
    106 }
    107 
    108 define i16 @test_atomic_load_sub_i16(i16 %offset) nounwind {
    109 ; CHECK-LABEL: test_atomic_load_sub_i16:
    110    %old = atomicrmw sub i16* @var16, i16 %offset release
    111 ; CHECK-NOT: dmb
    112 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    113 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    114 
    115 ; CHECK: .LBB{{[0-9]+}}_1:
    116 ; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    117   ; w0 below is a reasonable guess but could change: it certainly comes into the
    118   ;  function there.
    119 ; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
    120 ; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    121 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    122 ; CHECK-NOT: dmb
    123 
    124 ; CHECK: mov x0, x[[OLD]]
    125    ret i16 %old
    126 }
    127 
    128 define i32 @test_atomic_load_sub_i32(i32 %offset) nounwind {
    129 ; CHECK-LABEL: test_atomic_load_sub_i32:
    130    %old = atomicrmw sub i32* @var32, i32 %offset acquire
    131 ; CHECK-NOT: dmb
    132 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    133 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    134 
    135 ; CHECK: .LBB{{[0-9]+}}_1:
    136 ; CHECK-NEXT: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    137   ; w0 below is a reasonable guess but could change: it certainly comes into the
    138   ;  function there.
    139 ; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
    140 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    141 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    142 ; CHECK-NOT: dmb
    143 
    144 ; CHECK: mov x0, x[[OLD]]
    145    ret i32 %old
    146 }
    147 
    148 define i64 @test_atomic_load_sub_i64(i64 %offset) nounwind {
    149 ; CHECK-LABEL: test_atomic_load_sub_i64:
    150    %old = atomicrmw sub i64* @var64, i64 %offset seq_cst
    151 ; CHECK-NOT: dmb
    152 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    153 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    154 
    155 ; CHECK: .LBB{{[0-9]+}}_1:
    156 ; CHECK-NEXT: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    157   ; x0 below is a reasonable guess but could change: it certainly comes into the
    158   ; function there.
    159 ; CHECK-NEXT: sub [[NEW:x[0-9]+]], x[[OLD]], x0
    160 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    161 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    162 ; CHECK-NOT: dmb
    163 
    164 ; CHECK: mov x0, x[[OLD]]
    165    ret i64 %old
    166 }
    167 
    168 define i8 @test_atomic_load_and_i8(i8 %offset) nounwind {
    169 ; CHECK-LABEL: test_atomic_load_and_i8:
    170    %old = atomicrmw and i8* @var8, i8 %offset release
    171 ; CHECK-NOT: dmb
    172 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    173 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    174 
    175 ; CHECK: .LBB{{[0-9]+}}_1:
    176 ; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    177   ; w0 below is a reasonable guess but could change: it certainly comes into the
    178   ;  function there.
    179 ; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
    180 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    181 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    182 ; CHECK-NOT: dmb
    183 
    184 ; CHECK: mov x0, x[[OLD]]
    185    ret i8 %old
    186 }
    187 
    188 define i16 @test_atomic_load_and_i16(i16 %offset) nounwind {
    189 ; CHECK-LABEL: test_atomic_load_and_i16:
    190    %old = atomicrmw and i16* @var16, i16 %offset monotonic
    191 ; CHECK-NOT: dmb
    192 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    193 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    194 
    195 ; CHECK: .LBB{{[0-9]+}}_1:
    196 ; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    197   ; w0 below is a reasonable guess but could change: it certainly comes into the
    198   ;  function there.
    199 ; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
    200 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    201 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    202 ; CHECK-NOT: dmb
    203 
    204 ; CHECK: mov x0, x[[OLD]]
    205    ret i16 %old
    206 }
    207 
    208 define i32 @test_atomic_load_and_i32(i32 %offset) nounwind {
    209 ; CHECK-LABEL: test_atomic_load_and_i32:
    210    %old = atomicrmw and i32* @var32, i32 %offset seq_cst
    211 ; CHECK-NOT: dmb
    212 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    213 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    214 
    215 ; CHECK: .LBB{{[0-9]+}}_1:
    216 ; CHECK-NEXT: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    217   ; w0 below is a reasonable guess but could change: it certainly comes into the
    218   ;  function there.
    219 ; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
    220 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    221 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    222 ; CHECK-NOT: dmb
    223 
    224 ; CHECK: mov x0, x[[OLD]]
    225    ret i32 %old
    226 }
    227 
    228 define i64 @test_atomic_load_and_i64(i64 %offset) nounwind {
    229 ; CHECK-LABEL: test_atomic_load_and_i64:
    230    %old = atomicrmw and i64* @var64, i64 %offset acquire
    231 ; CHECK-NOT: dmb
    232 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    233 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    234 
    235 ; CHECK: .LBB{{[0-9]+}}_1:
    236 ; CHECK-NEXT: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    237   ; x0 below is a reasonable guess but could change: it certainly comes into the
    238   ; function there.
    239 ; CHECK-NEXT: and [[NEW:x[0-9]+]], x[[OLD]], x0
    240 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    241 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    242 ; CHECK-NOT: dmb
    243 
    244 ; CHECK: mov x0, x[[OLD]]
    245    ret i64 %old
    246 }
    247 
    248 define i8 @test_atomic_load_or_i8(i8 %offset) nounwind {
    249 ; CHECK-LABEL: test_atomic_load_or_i8:
    250    %old = atomicrmw or i8* @var8, i8 %offset seq_cst
    251 ; CHECK-NOT: dmb
    252 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    253 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    254 
    255 ; CHECK: .LBB{{[0-9]+}}_1:
    256 ; CHECK-NEXT: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    257   ; w0 below is a reasonable guess but could change: it certainly comes into the
    258   ;  function there.
    259 ; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
    260 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    261 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    262 ; CHECK-NOT: dmb
    263 
    264 ; CHECK: mov x0, x[[OLD]]
    265    ret i8 %old
    266 }
    267 
    268 define i16 @test_atomic_load_or_i16(i16 %offset) nounwind {
    269 ; CHECK-LABEL: test_atomic_load_or_i16:
    270    %old = atomicrmw or i16* @var16, i16 %offset monotonic
    271 ; CHECK-NOT: dmb
    272 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    273 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    274 
    275 ; CHECK: .LBB{{[0-9]+}}_1:
    276 ; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    277   ; w0 below is a reasonable guess but could change: it certainly comes into the
    278   ;  function there.
    279 ; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
    280 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    281 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    282 ; CHECK-NOT: dmb
    283 
    284 ; CHECK: mov x0, x[[OLD]]
    285    ret i16 %old
    286 }
    287 
    288 define i32 @test_atomic_load_or_i32(i32 %offset) nounwind {
    289 ; CHECK-LABEL: test_atomic_load_or_i32:
    290    %old = atomicrmw or i32* @var32, i32 %offset acquire
    291 ; CHECK-NOT: dmb
    292 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    293 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    294 
    295 ; CHECK: .LBB{{[0-9]+}}_1:
    296 ; CHECK-NEXT: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    297   ; w0 below is a reasonable guess but could change: it certainly comes into the
    298   ;  function there.
    299 ; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
    300 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    301 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    302 ; CHECK-NOT: dmb
    303 
    304 ; CHECK: mov x0, x[[OLD]]
    305    ret i32 %old
    306 }
    307 
    308 define i64 @test_atomic_load_or_i64(i64 %offset) nounwind {
    309 ; CHECK-LABEL: test_atomic_load_or_i64:
    310    %old = atomicrmw or i64* @var64, i64 %offset release
    311 ; CHECK-NOT: dmb
    312 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    313 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    314 
    315 ; CHECK: .LBB{{[0-9]+}}_1:
    316 ; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    317   ; x0 below is a reasonable guess but could change: it certainly comes into the
    318   ; function there.
    319 ; CHECK-NEXT: orr [[NEW:x[0-9]+]], x[[OLD]], x0
    320 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    321 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    322 ; CHECK-NOT: dmb
    323 
    324 ; CHECK: mov x0, x[[OLD]]
    325    ret i64 %old
    326 }
    327 
    328 define i8 @test_atomic_load_xor_i8(i8 %offset) nounwind {
    329 ; CHECK-LABEL: test_atomic_load_xor_i8:
    330    %old = atomicrmw xor i8* @var8, i8 %offset acquire
    331 ; CHECK-NOT: dmb
    332 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    333 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    334 
    335 ; CHECK: .LBB{{[0-9]+}}_1:
    336 ; CHECK-NEXT: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    337   ; w0 below is a reasonable guess but could change: it certainly comes into the
    338   ;  function there.
    339 ; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
    340 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    341 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    342 ; CHECK-NOT: dmb
    343 
    344 ; CHECK: mov x0, x[[OLD]]
    345    ret i8 %old
    346 }
    347 
    348 define i16 @test_atomic_load_xor_i16(i16 %offset) nounwind {
    349 ; CHECK-LABEL: test_atomic_load_xor_i16:
    350    %old = atomicrmw xor i16* @var16, i16 %offset release
    351 ; CHECK-NOT: dmb
    352 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    353 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    354 
    355 ; CHECK: .LBB{{[0-9]+}}_1:
    356 ; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    357   ; w0 below is a reasonable guess but could change: it certainly comes into the
    358   ;  function there.
    359 ; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
    360 ; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    361 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    362 ; CHECK-NOT: dmb
    363 
    364 ; CHECK: mov x0, x[[OLD]]
    365    ret i16 %old
    366 }
    367 
    368 define i32 @test_atomic_load_xor_i32(i32 %offset) nounwind {
    369 ; CHECK-LABEL: test_atomic_load_xor_i32:
    370    %old = atomicrmw xor i32* @var32, i32 %offset seq_cst
    371 ; CHECK-NOT: dmb
    372 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    373 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    374 
    375 ; CHECK: .LBB{{[0-9]+}}_1:
    376 ; CHECK-NEXT: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    377   ; w0 below is a reasonable guess but could change: it certainly comes into the
    378   ;  function there.
    379 ; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
    380 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    381 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    382 ; CHECK-NOT: dmb
    383 
    384 ; CHECK: mov x0, x[[OLD]]
    385    ret i32 %old
    386 }
    387 
    388 define i64 @test_atomic_load_xor_i64(i64 %offset) nounwind {
    389 ; CHECK-LABEL: test_atomic_load_xor_i64:
    390    %old = atomicrmw xor i64* @var64, i64 %offset monotonic
    391 ; CHECK-NOT: dmb
    392 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    393 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    394 
    395 ; CHECK: .LBB{{[0-9]+}}_1:
    396 ; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    397   ; x0 below is a reasonable guess but could change: it certainly comes into the
    398   ; function there.
    399 ; CHECK-NEXT: eor [[NEW:x[0-9]+]], x[[OLD]], x0
    400 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    401 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    402 ; CHECK-NOT: dmb
    403 
    404 ; CHECK: mov x0, x[[OLD]]
    405    ret i64 %old
    406 }
    407 
    408 define i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
    409 ; CHECK-LABEL: test_atomic_load_xchg_i8:
    410    %old = atomicrmw xchg i8* @var8, i8 %offset monotonic
    411 ; CHECK-NOT: dmb
    412 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    413 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    414 
    415 ; CHECK: .LBB{{[0-9]+}}_1:
    416 ; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    417   ; w0 below is a reasonable guess but could change: it certainly comes into the
    418   ; function there.
    419 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
    420 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    421 ; CHECK-NOT: dmb
    422 
    423 ; CHECK: mov x0, x[[OLD]]
    424    ret i8 %old
    425 }
    426 
    427 define i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind {
    428 ; CHECK-LABEL: test_atomic_load_xchg_i16:
    429    %old = atomicrmw xchg i16* @var16, i16 %offset seq_cst
    430 ; CHECK-NOT: dmb
    431 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    432 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    433 
    434 ; CHECK: .LBB{{[0-9]+}}_1:
    435 ; CHECK-NEXT: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    436   ; w0 below is a reasonable guess but could change: it certainly comes into the
    437   ; function there.
    438 ; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
    439 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    440 ; CHECK-NOT: dmb
    441 
    442 ; CHECK: mov x0, x[[OLD]]
    443    ret i16 %old
    444 }
    445 
    446 define i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind {
    447 ; CHECK-LABEL: test_atomic_load_xchg_i32:
    448    %old = atomicrmw xchg i32* @var32, i32 %offset release
    449 ; CHECK-NOT: dmb
    450 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    451 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    452 
    453 ; CHECK: .LBB{{[0-9]+}}_1:
    454 ; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    455   ; w0 below is a reasonable guess but could change: it certainly comes into the
    456   ;  function there.
    457 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
    458 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    459 ; CHECK-NOT: dmb
    460 
    461 ; CHECK: mov x0, x[[OLD]]
    462    ret i32 %old
    463 }
    464 
    465 define i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind {
    466 ; CHECK-LABEL: test_atomic_load_xchg_i64:
    467    %old = atomicrmw xchg i64* @var64, i64 %offset acquire
    468 ; CHECK-NOT: dmb
    469 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    470 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    471 
    472 ; CHECK: .LBB{{[0-9]+}}_1:
    473 ; CHECK-NEXT: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    474   ; x0 below is a reasonable guess but could change: it certainly comes into the
    475   ; function there.
    476 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], x0, [x[[ADDR]]]
    477 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    478 ; CHECK-NOT: dmb
    479 
    480 ; CHECK: mov x0, x[[OLD]]
    481    ret i64 %old
    482 }
    483 
    484 
    485 define i8 @test_atomic_load_min_i8(i8 %offset) nounwind {
    486 ; CHECK-LABEL: test_atomic_load_min_i8:
    487    %old = atomicrmw min i8* @var8, i8 %offset acquire
    488 ; CHECK-NOT: dmb
    489 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    490 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    491 
    492 ; CHECK: .LBB{{[0-9]+}}_1:
    493 ; CHECK-NEXT: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    494   ; w0 below is a reasonable guess but could change: it certainly comes into the
    495   ;  function there.
    496 ; CHECK-NEXT: cmp w0, w[[OLD]], sxtb
    497 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
    498 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    499 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    500 ; CHECK-NOT: dmb
    501 
    502 ; CHECK: mov x0, x[[OLD]]
    503    ret i8 %old
    504 }
    505 
    506 define i16 @test_atomic_load_min_i16(i16 %offset) nounwind {
    507 ; CHECK-LABEL: test_atomic_load_min_i16:
    508    %old = atomicrmw min i16* @var16, i16 %offset release
    509 ; CHECK-NOT: dmb
    510 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    511 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    512 
    513 ; CHECK: .LBB{{[0-9]+}}_1:
    514 ; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    515   ; w0 below is a reasonable guess but could change: it certainly comes into the
    516   ;  function there.
    517 ; CHECK-NEXT: cmp w0, w[[OLD]], sxth
    518 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
    519 ; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    520 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    521 ; CHECK-NOT: dmb
    522 
    523 ; CHECK: mov x0, x[[OLD]]
    524    ret i16 %old
    525 }
    526 
    527 define i32 @test_atomic_load_min_i32(i32 %offset) nounwind {
    528 ; CHECK-LABEL: test_atomic_load_min_i32:
    529    %old = atomicrmw min i32* @var32, i32 %offset monotonic
    530 ; CHECK-NOT: dmb
    531 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    532 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    533 
    534 ; CHECK: .LBB{{[0-9]+}}_1:
    535 ; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    536   ; w0 below is a reasonable guess but could change: it certainly comes into the
    537   ;  function there.
    538 ; CHECK-NEXT: cmp w0, w[[OLD]]
    539 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
    540 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    541 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    542 ; CHECK-NOT: dmb
    543 
    544 ; CHECK: mov x0, x[[OLD]]
    545    ret i32 %old
    546 }
    547 
    548 define i64 @test_atomic_load_min_i64(i64 %offset) nounwind {
    549 ; CHECK-LABEL: test_atomic_load_min_i64:
    550    %old = atomicrmw min i64* @var64, i64 %offset seq_cst
    551 ; CHECK-NOT: dmb
    552 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    553 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    554 
    555 ; CHECK: .LBB{{[0-9]+}}_1:
    556 ; CHECK-NEXT: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    557   ; x0 below is a reasonable guess but could change: it certainly comes into the
    558   ; function there.
    559 ; CHECK-NEXT: cmp x0, x[[OLD]]
    560 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, gt
    561 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    562 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    563 ; CHECK-NOT: dmb
    564 
    565 ; CHECK: mov x0, x[[OLD]]
    566    ret i64 %old
    567 }
    568 
    569 define i8 @test_atomic_load_max_i8(i8 %offset) nounwind {
    570 ; CHECK-LABEL: test_atomic_load_max_i8:
    571    %old = atomicrmw max i8* @var8, i8 %offset seq_cst
    572 ; CHECK-NOT: dmb
    573 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    574 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    575 
    576 ; CHECK: .LBB{{[0-9]+}}_1:
    577 ; CHECK-NEXT: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    578   ; w0 below is a reasonable guess but could change: it certainly comes into the
    579   ;  function there.
    580 ; CHECK-NEXT: cmp w0, w[[OLD]], sxtb
    581 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lt
    582 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    583 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    584 ; CHECK-NOT: dmb
    585 
    586 ; CHECK: mov x0, x[[OLD]]
    587    ret i8 %old
    588 }
    589 
    590 define i16 @test_atomic_load_max_i16(i16 %offset) nounwind {
    591 ; CHECK-LABEL: test_atomic_load_max_i16:
    592    %old = atomicrmw max i16* @var16, i16 %offset acquire
    593 ; CHECK-NOT: dmb
    594 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    595 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    596 
    597 ; CHECK: .LBB{{[0-9]+}}_1:
    598 ; CHECK-NEXT: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    599   ; w0 below is a reasonable guess but could change: it certainly comes into the
    600   ;  function there.
    601 ; CHECK-NEXT: cmp w0, w[[OLD]], sxth
    602 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lt
    603 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    604 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    605 ; CHECK-NOT: dmb
    606 
    607 ; CHECK: mov x0, x[[OLD]]
    608    ret i16 %old
    609 }
    610 
    611 define i32 @test_atomic_load_max_i32(i32 %offset) nounwind {
    612 ; CHECK-LABEL: test_atomic_load_max_i32:
    613    %old = atomicrmw max i32* @var32, i32 %offset release
    614 ; CHECK-NOT: dmb
    615 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    616 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    617 
    618 ; CHECK: .LBB{{[0-9]+}}_1:
    619 ; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    620   ; w0 below is a reasonable guess but could change: it certainly comes into the
    621   ;  function there.
    622 ; CHECK-NEXT: cmp w0, w[[OLD]]
    623 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lt
    624 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    625 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    626 ; CHECK-NOT: dmb
    627 
    628 ; CHECK: mov x0, x[[OLD]]
    629    ret i32 %old
    630 }
    631 
    632 define i64 @test_atomic_load_max_i64(i64 %offset) nounwind {
    633 ; CHECK-LABEL: test_atomic_load_max_i64:
    634    %old = atomicrmw max i64* @var64, i64 %offset monotonic
    635 ; CHECK-NOT: dmb
    636 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    637 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    638 
    639 ; CHECK: .LBB{{[0-9]+}}_1:
    640 ; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    641   ; x0 below is a reasonable guess but could change: it certainly comes into the
    642   ; function there.
    643 ; CHECK-NEXT: cmp x0, x[[OLD]]
    644 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, lt
    645 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    646 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    647 ; CHECK-NOT: dmb
    648 
    649 ; CHECK: mov x0, x[[OLD]]
    650    ret i64 %old
    651 }
    652 
    653 define i8 @test_atomic_load_umin_i8(i8 %offset) nounwind {
    654 ; CHECK-LABEL: test_atomic_load_umin_i8:
    655    %old = atomicrmw umin i8* @var8, i8 %offset monotonic
    656 ; CHECK-NOT: dmb
    657 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    658 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    659 
    660 ; CHECK: .LBB{{[0-9]+}}_1:
    661 ; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    662   ; w0 below is a reasonable guess but could change: it certainly comes into the
    663   ;  function there.
    664 ; CHECK-NEXT: cmp w0, w[[OLD]], uxtb
    665 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
    666 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    667 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    668 ; CHECK-NOT: dmb
    669 
    670 ; CHECK: mov x0, x[[OLD]]
    671    ret i8 %old
    672 }
    673 
    674 define i16 @test_atomic_load_umin_i16(i16 %offset) nounwind {
    675 ; CHECK-LABEL: test_atomic_load_umin_i16:
    676    %old = atomicrmw umin i16* @var16, i16 %offset acquire
    677 ; CHECK-NOT: dmb
    678 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    679 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    680 
    681 ; CHECK: .LBB{{[0-9]+}}_1:
    682 ; CHECK-NEXT: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    683   ; w0 below is a reasonable guess but could change: it certainly comes into the
    684   ;  function there.
    685 ; CHECK-NEXT: cmp w0, w[[OLD]], uxth
    686 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
    687 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    688 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    689 ; CHECK-NOT: dmb
    690 
    691 ; CHECK: mov x0, x[[OLD]]
    692    ret i16 %old
    693 }
    694 
    695 define i32 @test_atomic_load_umin_i32(i32 %offset) nounwind {
    696 ; CHECK-LABEL: test_atomic_load_umin_i32:
    697    %old = atomicrmw umin i32* @var32, i32 %offset seq_cst
    698 ; CHECK-NOT: dmb
    699 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    700 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    701 
    702 ; CHECK: .LBB{{[0-9]+}}_1:
    703 ; CHECK-NEXT: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    704   ; w0 below is a reasonable guess but could change: it certainly comes into the
    705   ;  function there.
    706 ; CHECK-NEXT: cmp w0, w[[OLD]]
    707 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
    708 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    709 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    710 ; CHECK-NOT: dmb
    711 
    712 ; CHECK: mov x0, x[[OLD]]
    713    ret i32 %old
    714 }
    715 
    716 define i64 @test_atomic_load_umin_i64(i64 %offset) nounwind {
    717 ; CHECK-LABEL: test_atomic_load_umin_i64:
    718    %old = atomicrmw umin i64* @var64, i64 %offset acq_rel
    719 ; CHECK-NOT: dmb
    720 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    721 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    722 
    723 ; CHECK: .LBB{{[0-9]+}}_1:
    724 ; CHECK-NEXT: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    725   ; x0 below is a reasonable guess but could change: it certainly comes into the
    726   ; function there.
    727 ; CHECK-NEXT: cmp x0, x[[OLD]]
    728 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, hi
    729 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    730 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    731 ; CHECK-NOT: dmb
    732 
    733 ; CHECK: mov x0, x[[OLD]]
    734    ret i64 %old
    735 }
    736 
    737 define i8 @test_atomic_load_umax_i8(i8 %offset) nounwind {
    738 ; CHECK-LABEL: test_atomic_load_umax_i8:
    739    %old = atomicrmw umax i8* @var8, i8 %offset acq_rel
    740 ; CHECK-NOT: dmb
    741 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    742 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    743 
    744 ; CHECK: .LBB{{[0-9]+}}_1:
    745 ; CHECK-NEXT: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    746   ; w0 below is a reasonable guess but could change: it certainly comes into the
    747   ;  function there.
    748 ; CHECK-NEXT: cmp w0, w[[OLD]], uxtb
    749 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lo
    750 ; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    751 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    752 ; CHECK-NOT: dmb
    753 
    754 ; CHECK: mov x0, x[[OLD]]
    755    ret i8 %old
    756 }
    757 
    758 define i16 @test_atomic_load_umax_i16(i16 %offset) nounwind {
    759 ; CHECK-LABEL: test_atomic_load_umax_i16:
    760    %old = atomicrmw umax i16* @var16, i16 %offset monotonic
    761 ; CHECK-NOT: dmb
    762 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    763 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    764 
    765 ; CHECK: .LBB{{[0-9]+}}_1:
    766 ; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    767   ; w0 below is a reasonable guess but could change: it certainly comes into the
    768   ;  function there.
    769 ; CHECK-NEXT: cmp w0, w[[OLD]], uxth
    770 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lo
    771 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    772 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    773 ; CHECK-NOT: dmb
    774 
    775 ; CHECK: mov x0, x[[OLD]]
    776    ret i16 %old
    777 }
    778 
    779 define i32 @test_atomic_load_umax_i32(i32 %offset) nounwind {
    780 ; CHECK-LABEL: test_atomic_load_umax_i32:
    781    %old = atomicrmw umax i32* @var32, i32 %offset seq_cst
    782 ; CHECK-NOT: dmb
    783 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    784 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    785 
    786 ; CHECK: .LBB{{[0-9]+}}_1:
    787 ; CHECK-NEXT: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    788   ; w0 below is a reasonable guess but could change: it certainly comes into the
    789   ;  function there.
    790 ; CHECK-NEXT: cmp w0, w[[OLD]]
    791 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lo
    792 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    793 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    794 ; CHECK-NOT: dmb
    795 
    796 ; CHECK: mov x0, x[[OLD]]
    797    ret i32 %old
    798 }
    799 
    800 define i64 @test_atomic_load_umax_i64(i64 %offset) nounwind {
    801 ; CHECK-LABEL: test_atomic_load_umax_i64:
    802    %old = atomicrmw umax i64* @var64, i64 %offset release
    803 ; CHECK-NOT: dmb
    804 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    805 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    806 
    807 ; CHECK: .LBB{{[0-9]+}}_1:
    808 ; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    809   ; x0 below is a reasonable guess but could change: it certainly comes into the
    810   ; function there.
    811 ; CHECK-NEXT: cmp x0, x[[OLD]]
    812 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, lo
    813 ; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
    814 ; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
    815 ; CHECK-NOT: dmb
    816 
    817 ; CHECK: mov x0, x[[OLD]]
    818    ret i64 %old
    819 }
    820 
    821 define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
    822 ; CHECK-LABEL: test_atomic_cmpxchg_i8:
    823    %old = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire
    824 ; CHECK-NOT: dmb
    825 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    826 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    827 
    828 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
    829 ; CHECK-NEXT: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
    830   ; w0 below is a reasonable guess but could change: it certainly comes into the
    831   ;  function there.
    832 ; CHECK-NEXT: cmp w[[OLD]], w0
    833 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
    834   ; As above, w1 is a reasonable guess.
    835 ; CHECK: stxrb [[STATUS:w[0-9]+]], w1, [x[[ADDR]]]
    836 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
    837 ; CHECK-NOT: dmb
    838 
    839 ; CHECK: mov x0, x[[OLD]]
    840    ret i8 %old
    841 }
    842 
    843 define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
    844 ; CHECK-LABEL: test_atomic_cmpxchg_i16:
    845    %old = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst
    846 ; CHECK-NOT: dmb
    847 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
    848 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
    849 
    850 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
    851 ; CHECK-NEXT: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
    852   ; w0 below is a reasonable guess but could change: it certainly comes into the
    853   ;  function there.
    854 ; CHECK-NEXT: cmp w[[OLD]], w0
    855 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
    856   ; As above, w1 is a reasonable guess.
    857 ; CHECK: stlxrh [[STATUS:w[0-9]+]], w1, [x[[ADDR]]]
    858 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
    859 ; CHECK-NOT: dmb
    860 
    861 ; CHECK: mov x0, x[[OLD]]
    862    ret i16 %old
    863 }
    864 
    865 define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
    866 ; CHECK-LABEL: test_atomic_cmpxchg_i32:
    867    %old = cmpxchg i32* @var32, i32 %wanted, i32 %new release
    868 ; CHECK-NOT: dmb
    869 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
    870 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
    871 
    872 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
    873 ; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
    874   ; w0 below is a reasonable guess but could change: it certainly comes into the
    875   ;  function there.
    876 ; CHECK-NEXT: cmp w[[OLD]], w0
    877 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
    878   ; As above, w1 is a reasonable guess.
    879 ; CHECK: stlxr [[STATUS:w[0-9]+]], w1, [x[[ADDR]]]
    880 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
    881 ; CHECK-NOT: dmb
    882 
    883 ; CHECK: mov x0, x[[OLD]]
    884    ret i32 %old
    885 }
    886 
    887 define i64 @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
    888 ; CHECK-LABEL: test_atomic_cmpxchg_i64:
    889    %old = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic
    890 ; CHECK-NOT: dmb
    891 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
    892 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
    893 
    894 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
    895 ; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
    896   ; w0 below is a reasonable guess but could change: it certainly comes into the
    897   ;  function there.
    898 ; CHECK-NEXT: cmp x[[OLD]], x0
    899 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
    900   ; As above, w1 is a reasonable guess.
    901 ; CHECK: stxr [[STATUS:w[0-9]+]], x1, [x[[ADDR]]]
    902 ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
    903 ; CHECK-NOT: dmb
    904 
    905 ; CHECK: mov x0, x[[OLD]]
    906    ret i64 %old
    907 }
    908 
    909 define i8 @test_atomic_load_monotonic_i8() nounwind {
    910 ; CHECK-LABEL: test_atomic_load_monotonic_i8:
    911   %val = load atomic i8* @var8 monotonic, align 1
    912 ; CHECK-NOT: dmb
    913 ; CHECK: adrp x[[HIADDR:[0-9]+]], var8
    914 ; CHECK: ldrb w0, [x[[HIADDR]], #:lo12:var8]
    915 ; CHECK-NOT: dmb
    916 
    917   ret i8 %val
    918 }
    919 
    920 define i8 @test_atomic_load_monotonic_regoff_i8(i64 %base, i64 %off) nounwind {
    921 ; CHECK-LABEL: test_atomic_load_monotonic_regoff_i8:
    922   %addr_int = add i64 %base, %off
    923   %addr = inttoptr i64 %addr_int to i8*
    924 
    925   %val = load atomic i8* %addr monotonic, align 1
    926 ; CHECK-NOT: dmb
    927 ; CHECK: ldrb w0, [x0, x1]
    928 ; CHECK-NOT: dmb
    929 
    930   ret i8 %val
    931 }
    932 
    933 define i8 @test_atomic_load_acquire_i8() nounwind {
    934 ; CHECK-LABEL: test_atomic_load_acquire_i8:
    935   %val = load atomic i8* @var8 acquire, align 1
    936 ; CHECK-NOT: dmb
    937 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
    938 ; CHECK-NOT: dmb
    939 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
    940 ; CHECK-NOT: dmb
    941 ; CHECK: ldarb w0, [x[[ADDR]]]
    942 ; CHECK-NOT: dmb
    943   ret i8 %val
    944 }
    945 
    946 define i8 @test_atomic_load_seq_cst_i8() nounwind {
    947 ; CHECK-LABEL: test_atomic_load_seq_cst_i8:
    948   %val = load atomic i8* @var8 seq_cst, align 1
    949 ; CHECK-NOT: dmb
    950 ; CHECK: adrp [[HIADDR:x[0-9]+]], var8
    951 ; CHECK-NOT: dmb
    952 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], #:lo12:var8
    953 ; CHECK-NOT: dmb
    954 ; CHECK: ldarb w0, [x[[ADDR]]]
    955 ; CHECK-NOT: dmb
    956   ret i8 %val
    957 }
    958 
    959 define i16 @test_atomic_load_monotonic_i16() nounwind {
    960 ; CHECK-LABEL: test_atomic_load_monotonic_i16:
    961   %val = load atomic i16* @var16 monotonic, align 2
    962 ; CHECK-NOT: dmb
    963 ; CHECK: adrp x[[HIADDR:[0-9]+]], var16
    964 ; CHECK-NOT: dmb
    965 ; CHECK: ldrh w0, [x[[HIADDR]], #:lo12:var16]
    966 ; CHECK-NOT: dmb
    967 
    968   ret i16 %val
    969 }
    970 
    971 define i32 @test_atomic_load_monotonic_regoff_i32(i64 %base, i64 %off) nounwind {
    972 ; CHECK-LABEL: test_atomic_load_monotonic_regoff_i32:
    973   %addr_int = add i64 %base, %off
    974   %addr = inttoptr i64 %addr_int to i32*
    975 
    976   %val = load atomic i32* %addr monotonic, align 4
    977 ; CHECK-NOT: dmb
    978 ; CHECK: ldr w0, [x0, x1]
    979 ; CHECK-NOT: dmb
    980 
    981   ret i32 %val
    982 }
    983 
    984 define i64 @test_atomic_load_seq_cst_i64() nounwind {
    985 ; CHECK-LABEL: test_atomic_load_seq_cst_i64:
    986   %val = load atomic i64* @var64 seq_cst, align 8
    987 ; CHECK-NOT: dmb
    988 ; CHECK: adrp [[HIADDR:x[0-9]+]], var64
    989 ; CHECK-NOT: dmb
    990 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], #:lo12:var64
    991 ; CHECK-NOT: dmb
    992 ; CHECK: ldar x0, [x[[ADDR]]]
    993 ; CHECK-NOT: dmb
    994   ret i64 %val
    995 }
    996 
    997 define void @test_atomic_store_monotonic_i8(i8 %val) nounwind {
    998 ; CHECK-LABEL: test_atomic_store_monotonic_i8:
    999   store atomic i8 %val, i8* @var8 monotonic, align 1
   1000 ; CHECK: adrp x[[HIADDR:[0-9]+]], var8
   1001 ; CHECK: strb w0, [x[[HIADDR]], #:lo12:var8]
   1002 
   1003   ret void
   1004 }
   1005 
   1006 define void @test_atomic_store_monotonic_regoff_i8(i64 %base, i64 %off, i8 %val) nounwind {
   1007 ; CHECK-LABEL: test_atomic_store_monotonic_regoff_i8:
   1008 
   1009   %addr_int = add i64 %base, %off
   1010   %addr = inttoptr i64 %addr_int to i8*
   1011 
   1012   store atomic i8 %val, i8* %addr monotonic, align 1
   1013 ; CHECK: strb w2, [x0, x1]
   1014 
   1015   ret void
   1016 }
   1017 define void @test_atomic_store_release_i8(i8 %val) nounwind {
   1018 ; CHECK-LABEL: test_atomic_store_release_i8:
   1019   store atomic i8 %val, i8* @var8 release, align 1
   1020 ; CHECK-NOT: dmb
   1021 ; CHECK: adrp [[HIADDR:x[0-9]+]], var8
   1022 ; CHECK-NOT: dmb
   1023 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], #:lo12:var8
   1024 ; CHECK-NOT: dmb
   1025 ; CHECK: stlrb w0, [x[[ADDR]]]
   1026 ; CHECK-NOT: dmb
   1027   ret void
   1028 }
   1029 
   1030 define void @test_atomic_store_seq_cst_i8(i8 %val) nounwind {
   1031 ; CHECK-LABEL: test_atomic_store_seq_cst_i8:
   1032   store atomic i8 %val, i8* @var8 seq_cst, align 1
   1033 ; CHECK-NOT: dmb
   1034 ; CHECK: adrp [[HIADDR:x[0-9]+]], var8
   1035 ; CHECK-NOT: dmb
   1036 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], #:lo12:var8
   1037 ; CHECK-NOT: dmb
   1038 ; CHECK: stlrb w0, [x[[ADDR]]]
   1039 ; CHECK-NOT: dmb
   1040 
   1041   ret void
   1042 }
   1043 
   1044 define void @test_atomic_store_monotonic_i16(i16 %val) nounwind {
   1045 ; CHECK-LABEL: test_atomic_store_monotonic_i16:
   1046   store atomic i16 %val, i16* @var16 monotonic, align 2
   1047 ; CHECK-NOT: dmb
   1048 ; CHECK: adrp x[[HIADDR:[0-9]+]], var16
   1049 ; CHECK-NOT: dmb
   1050 ; CHECK: strh w0, [x[[HIADDR]], #:lo12:var16]
   1051 ; CHECK-NOT: dmb
   1052   ret void
   1053 }
   1054 
   1055 define void @test_atomic_store_monotonic_regoff_i32(i64 %base, i64 %off, i32 %val) nounwind {
   1056 ; CHECK-LABEL: test_atomic_store_monotonic_regoff_i32:
   1057 
   1058   %addr_int = add i64 %base, %off
   1059   %addr = inttoptr i64 %addr_int to i32*
   1060 
   1061   store atomic i32 %val, i32* %addr monotonic, align 4
   1062 ; CHECK-NOT: dmb
   1063 ; CHECK: str w2, [x0, x1]
   1064 ; CHECK-NOT: dmb
   1065 
   1066   ret void
   1067 }
   1068 
   1069 define void @test_atomic_store_release_i64(i64 %val) nounwind {
   1070 ; CHECK-LABEL: test_atomic_store_release_i64:
   1071   store atomic i64 %val, i64* @var64 release, align 8
   1072 ; CHECK-NOT: dmb
   1073 ; CHECK: adrp [[HIADDR:x[0-9]+]], var64
   1074 ; CHECK-NOT: dmb
   1075 ; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], #:lo12:var64
   1076 ; CHECK-NOT: dmb
   1077 ; CHECK: stlr x0, [x[[ADDR]]]
   1078 ; CHECK-NOT: dmb
   1079   ret void
   1080 }
   1081