1 ; RUN: llc < %s -march=arm64 -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=cyclone | FileCheck %s 2 3 @var = global i128 0 4 5 define i128 @val_compare_and_swap(i128* %p, i128 %oldval, i128 %newval) { 6 ; CHECK-LABEL: val_compare_and_swap: 7 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 8 ; CHECK: ldaxp [[RESULTLO:x[0-9]+]], [[RESULTHI:x[0-9]+]], [x[[ADDR:[0-9]+]]] 9 ; CHECK-DAG: eor [[MISMATCH_LO:x[0-9]+]], [[RESULTLO]], x2 10 ; CHECK-DAG: eor [[MISMATCH_HI:x[0-9]+]], [[RESULTHI]], x3 11 ; CHECK: orr [[MISMATCH:x[0-9]+]], [[MISMATCH_LO]], [[MISMATCH_HI]] 12 ; CHECK: cbnz [[MISMATCH]], [[DONE:.LBB[0-9]+_[0-9]+]] 13 ; CHECK: stxp [[SCRATCH_RES:w[0-9]+]], x4, x5, [x[[ADDR]]] 14 ; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 15 ; CHECK: [[DONE]]: 16 %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval acquire acquire 17 %val = extractvalue { i128, i1 } %pair, 0 18 ret i128 %val 19 } 20 21 define void @fetch_and_nand(i128* %p, i128 %bits) { 22 ; CHECK-LABEL: fetch_and_nand: 23 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 24 ; CHECK: ldxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 25 ; CHECK-DAG: and [[TMP_REGLO:x[0-9]+]], [[DEST_REGLO]], x2 26 ; CHECK-DAG: and [[TMP_REGHI:x[0-9]+]], [[DEST_REGHI]], x3 27 ; CHECK-DAG: mvn [[SCRATCH_REGLO:x[0-9]+]], [[TMP_REGLO]] 28 ; CHECK-DAG: mvn [[SCRATCH_REGHI:x[0-9]+]], [[TMP_REGHI]] 29 ; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 30 ; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 31 32 ; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] 33 %val = atomicrmw nand i128* %p, i128 %bits release 34 store i128 %val, i128* @var, align 16 35 ret void 36 } 37 38 define void @fetch_and_or(i128* %p, i128 %bits) { 39 ; CHECK-LABEL: fetch_and_or: 40 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 41 ; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 42 ; CHECK-DAG: orr [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2 43 ; CHECK-DAG: orr [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3 44 ; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 45 ; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 46 47 ; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] 48 %val = atomicrmw or i128* %p, i128 %bits seq_cst 49 store i128 %val, i128* @var, align 16 50 ret void 51 } 52 53 define void @fetch_and_add(i128* %p, i128 %bits) { 54 ; CHECK-LABEL: fetch_and_add: 55 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 56 ; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 57 ; CHECK: adds [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2 58 ; CHECK: adcs [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3 59 ; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 60 ; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 61 62 ; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] 63 %val = atomicrmw add i128* %p, i128 %bits seq_cst 64 store i128 %val, i128* @var, align 16 65 ret void 66 } 67 68 define void @fetch_and_sub(i128* %p, i128 %bits) { 69 ; CHECK-LABEL: fetch_and_sub: 70 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 71 ; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 72 ; CHECK: subs [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2 73 ; CHECK: sbcs [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3 74 ; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 75 ; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 76 77 ; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] 78 %val = atomicrmw sub i128* %p, i128 %bits seq_cst 79 store i128 %val, i128* @var, align 16 80 ret void 81 } 82 83 define void @fetch_and_min(i128* %p, i128 %bits) { 84 ; CHECK-LABEL: fetch_and_min: 85 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 86 ; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 87 ; CHECK: cmp [[DEST_REGLO]], x2 88 ; CHECK: cset [[LOCMP:w[0-9]+]], ls 89 ; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3 90 ; CHECK: cset [[HICMP:w[0-9]+]], le 91 ; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq 92 ; CHECK: cmp [[CMP]], #0 93 ; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne 94 ; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne 95 ; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 96 ; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 97 98 ; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] 99 %val = atomicrmw min i128* %p, i128 %bits seq_cst 100 store i128 %val, i128* @var, align 16 101 ret void 102 } 103 104 define void @fetch_and_max(i128* %p, i128 %bits) { 105 ; CHECK-LABEL: fetch_and_max: 106 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 107 ; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 108 ; CHECK: cmp [[DEST_REGLO]], x2 109 ; CHECK: cset [[LOCMP:w[0-9]+]], hi 110 ; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3 111 ; CHECK: cset [[HICMP:w[0-9]+]], gt 112 ; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq 113 ; CHECK: cmp [[CMP]], #0 114 ; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne 115 ; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne 116 ; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 117 ; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 118 119 ; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] 120 %val = atomicrmw max i128* %p, i128 %bits seq_cst 121 store i128 %val, i128* @var, align 16 122 ret void 123 } 124 125 define void @fetch_and_umin(i128* %p, i128 %bits) { 126 ; CHECK-LABEL: fetch_and_umin: 127 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 128 ; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 129 ; CHECK: cmp [[DEST_REGLO]], x2 130 ; CHECK: cset [[LOCMP:w[0-9]+]], ls 131 ; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3 132 ; CHECK: cset [[HICMP:w[0-9]+]], ls 133 ; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq 134 ; CHECK: cmp [[CMP]], #0 135 ; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne 136 ; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne 137 ; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 138 ; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 139 140 ; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] 141 %val = atomicrmw umin i128* %p, i128 %bits seq_cst 142 store i128 %val, i128* @var, align 16 143 ret void 144 } 145 146 define void @fetch_and_umax(i128* %p, i128 %bits) { 147 ; CHECK-LABEL: fetch_and_umax: 148 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 149 ; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 150 ; CHECK: cmp [[DEST_REGLO]], x2 151 ; CHECK: cset [[LOCMP:w[0-9]+]], hi 152 ; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3 153 ; CHECK: cset [[HICMP:w[0-9]+]], hi 154 ; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq 155 ; CHECK: cmp [[CMP]], #0 156 ; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne 157 ; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne 158 ; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 159 ; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 160 161 ; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] 162 %val = atomicrmw umax i128* %p, i128 %bits seq_cst 163 store i128 %val, i128* @var, align 16 164 ret void 165 } 166 167 define i128 @atomic_load_seq_cst(i128* %p) { 168 ; CHECK-LABEL: atomic_load_seq_cst: 169 ; CHECK-NOT: dmb 170 ; CHECK-LABEL: ldaxp 171 ; CHECK-NOT: dmb 172 %r = load atomic i128, i128* %p seq_cst, align 16 173 ret i128 %r 174 } 175 176 define i128 @atomic_load_relaxed(i64, i64, i128* %p) { 177 ; CHECK-LABEL: atomic_load_relaxed: 178 ; CHECK-NOT: dmb 179 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 180 ; CHECK: ldxp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x2] 181 ; CHECK-NEXT: stxp [[SUCCESS:w[0-9]+]], [[LO]], [[HI]], [x2] 182 ; CHECK: cbnz [[SUCCESS]], [[LABEL]] 183 ; CHECK-NOT: dmb 184 %r = load atomic i128, i128* %p monotonic, align 16 185 ret i128 %r 186 } 187 188 189 define void @atomic_store_seq_cst(i128 %in, i128* %p) { 190 ; CHECK-LABEL: atomic_store_seq_cst: 191 ; CHECK-NOT: dmb 192 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 193 ; CHECK: ldaxp xzr, [[IGNORED:x[0-9]+]], [x2] 194 ; CHECK: stlxp [[SUCCESS:w[0-9]+]], x0, x1, [x2] 195 ; CHECK: cbnz [[SUCCESS]], [[LABEL]] 196 ; CHECK-NOT: dmb 197 store atomic i128 %in, i128* %p seq_cst, align 16 198 ret void 199 } 200 201 define void @atomic_store_release(i128 %in, i128* %p) { 202 ; CHECK-LABEL: atomic_store_release: 203 ; CHECK-NOT: dmb 204 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 205 ; CHECK: ldxp xzr, [[IGNORED:x[0-9]+]], [x2] 206 ; CHECK: stlxp [[SUCCESS:w[0-9]+]], x0, x1, [x2] 207 ; CHECK: cbnz [[SUCCESS]], [[LABEL]] 208 ; CHECK-NOT: dmb 209 store atomic i128 %in, i128* %p release, align 16 210 ret void 211 } 212 213 define void @atomic_store_relaxed(i128 %in, i128* %p) { 214 ; CHECK-LABEL: atomic_store_relaxed: 215 ; CHECK-NOT: dmb 216 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 217 ; CHECK: ldxp xzr, [[IGNORED:x[0-9]+]], [x2] 218 ; CHECK: stxp [[SUCCESS:w[0-9]+]], x0, x1, [x2] 219 ; CHECK: cbnz [[SUCCESS]], [[LABEL]] 220 ; CHECK-NOT: dmb 221 store atomic i128 %in, i128* %p unordered, align 16 222 ret void 223 } 224
