1 //=- AArch64InstrAtomics.td - AArch64 Atomic codegen support -*- tablegen -*-=// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // AArch64 Atomic operand code-gen constructs. 11 // 12 //===----------------------------------------------------------------------===// 13 14 //===---------------------------------- 15 // Atomic fences 16 //===---------------------------------- 17 def : Pat<(atomic_fence (i64 4), (imm)), (DMB (i32 0x9))>; 18 def : Pat<(atomic_fence (imm), (imm)), (DMB (i32 0xb))>; 19 20 //===---------------------------------- 21 // Atomic loads 22 //===---------------------------------- 23 24 // When they're actually atomic, only one addressing mode (GPR64sp) is 25 // supported, but when they're relaxed and anything can be used, all the 26 // standard modes would be valid and may give efficiency gains. 27 28 // A atomic load operation that actually needs acquire semantics. 29 class acquiring_load<PatFrag base> 30 : PatFrag<(ops node:$ptr), (base node:$ptr), [{ 31 AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering(); 32 assert(Ordering != AcquireRelease && "unexpected load ordering"); 33 return Ordering == Acquire || Ordering == SequentiallyConsistent; 34 }]>; 35 36 // An atomic load operation that does not need either acquire or release 37 // semantics. 38 class relaxed_load<PatFrag base> 39 : PatFrag<(ops node:$ptr), (base node:$ptr), [{ 40 AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering(); 41 return Ordering == Monotonic || Ordering == Unordered; 42 }]>; 43 44 // 8-bit loads 45 def : Pat<(acquiring_load<atomic_load_8> GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>; 46 def : Pat<(relaxed_load<atomic_load_8> (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm, 47 ro_Wextend8:$offset)), 48 (LDRBBroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$offset)>; 49 def : Pat<(relaxed_load<atomic_load_8> (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm, 50 ro_Xextend8:$offset)), 51 (LDRBBroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$offset)>; 52 def : Pat<(relaxed_load<atomic_load_8> (am_indexed8 GPR64sp:$Rn, 53 uimm12s1:$offset)), 54 (LDRBBui GPR64sp:$Rn, uimm12s1:$offset)>; 55 def : Pat<(relaxed_load<atomic_load_8> 56 (am_unscaled8 GPR64sp:$Rn, simm9:$offset)), 57 (LDURBBi GPR64sp:$Rn, simm9:$offset)>; 58 59 // 16-bit loads 60 def : Pat<(acquiring_load<atomic_load_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>; 61 def : Pat<(relaxed_load<atomic_load_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm, 62 ro_Wextend16:$extend)), 63 (LDRHHroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>; 64 def : Pat<(relaxed_load<atomic_load_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm, 65 ro_Xextend16:$extend)), 66 (LDRHHroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>; 67 def : Pat<(relaxed_load<atomic_load_16> (am_indexed16 GPR64sp:$Rn, 68 uimm12s2:$offset)), 69 (LDRHHui GPR64sp:$Rn, uimm12s2:$offset)>; 70 def : Pat<(relaxed_load<atomic_load_16> 71 (am_unscaled16 GPR64sp:$Rn, simm9:$offset)), 72 (LDURHHi GPR64sp:$Rn, simm9:$offset)>; 73 74 // 32-bit loads 75 def : Pat<(acquiring_load<atomic_load_32> GPR64sp:$ptr), (LDARW GPR64sp:$ptr)>; 76 def : Pat<(relaxed_load<atomic_load_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, 77 ro_Wextend32:$extend)), 78 (LDRWroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>; 79 def : Pat<(relaxed_load<atomic_load_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, 80 ro_Xextend32:$extend)), 81 (LDRWroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>; 82 def : Pat<(relaxed_load<atomic_load_32> (am_indexed32 GPR64sp:$Rn, 83 uimm12s4:$offset)), 84 (LDRWui GPR64sp:$Rn, uimm12s4:$offset)>; 85 def : Pat<(relaxed_load<atomic_load_32> 86 (am_unscaled32 GPR64sp:$Rn, simm9:$offset)), 87 (LDURWi GPR64sp:$Rn, simm9:$offset)>; 88 89 // 64-bit loads 90 def : Pat<(acquiring_load<atomic_load_64> GPR64sp:$ptr), (LDARX GPR64sp:$ptr)>; 91 def : Pat<(relaxed_load<atomic_load_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, 92 ro_Wextend64:$extend)), 93 (LDRXroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>; 94 def : Pat<(relaxed_load<atomic_load_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, 95 ro_Xextend64:$extend)), 96 (LDRXroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>; 97 def : Pat<(relaxed_load<atomic_load_64> (am_indexed64 GPR64sp:$Rn, 98 uimm12s8:$offset)), 99 (LDRXui GPR64sp:$Rn, uimm12s8:$offset)>; 100 def : Pat<(relaxed_load<atomic_load_64> 101 (am_unscaled64 GPR64sp:$Rn, simm9:$offset)), 102 (LDURXi GPR64sp:$Rn, simm9:$offset)>; 103 104 //===---------------------------------- 105 // Atomic stores 106 //===---------------------------------- 107 108 // When they're actually atomic, only one addressing mode (GPR64sp) is 109 // supported, but when they're relaxed and anything can be used, all the 110 // standard modes would be valid and may give efficiency gains. 111 112 // A store operation that actually needs release semantics. 113 class releasing_store<PatFrag base> 114 : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{ 115 AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering(); 116 assert(Ordering != AcquireRelease && "unexpected store ordering"); 117 return Ordering == Release || Ordering == SequentiallyConsistent; 118 }]>; 119 120 // An atomic store operation that doesn't actually need to be atomic on AArch64. 121 class relaxed_store<PatFrag base> 122 : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{ 123 AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering(); 124 return Ordering == Monotonic || Ordering == Unordered; 125 }]>; 126 127 // 8-bit stores 128 def : Pat<(releasing_store<atomic_store_8> GPR64sp:$ptr, GPR32:$val), 129 (STLRB GPR32:$val, GPR64sp:$ptr)>; 130 def : Pat<(relaxed_store<atomic_store_8> 131 (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend), 132 GPR32:$val), 133 (STRBBroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend)>; 134 def : Pat<(relaxed_store<atomic_store_8> 135 (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend), 136 GPR32:$val), 137 (STRBBroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend)>; 138 def : Pat<(relaxed_store<atomic_store_8> 139 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset), GPR32:$val), 140 (STRBBui GPR32:$val, GPR64sp:$Rn, uimm12s1:$offset)>; 141 def : Pat<(relaxed_store<atomic_store_8> 142 (am_unscaled8 GPR64sp:$Rn, simm9:$offset), GPR32:$val), 143 (STURBBi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>; 144 145 // 16-bit stores 146 def : Pat<(releasing_store<atomic_store_16> GPR64sp:$ptr, GPR32:$val), 147 (STLRH GPR32:$val, GPR64sp:$ptr)>; 148 def : Pat<(relaxed_store<atomic_store_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm, 149 ro_Wextend16:$extend), 150 GPR32:$val), 151 (STRHHroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>; 152 def : Pat<(relaxed_store<atomic_store_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm, 153 ro_Xextend16:$extend), 154 GPR32:$val), 155 (STRHHroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>; 156 def : Pat<(relaxed_store<atomic_store_16> 157 (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset), GPR32:$val), 158 (STRHHui GPR32:$val, GPR64sp:$Rn, uimm12s2:$offset)>; 159 def : Pat<(relaxed_store<atomic_store_16> 160 (am_unscaled16 GPR64sp:$Rn, simm9:$offset), GPR32:$val), 161 (STURHHi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>; 162 163 // 32-bit stores 164 def : Pat<(releasing_store<atomic_store_32> GPR64sp:$ptr, GPR32:$val), 165 (STLRW GPR32:$val, GPR64sp:$ptr)>; 166 def : Pat<(relaxed_store<atomic_store_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, 167 ro_Wextend32:$extend), 168 GPR32:$val), 169 (STRWroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>; 170 def : Pat<(relaxed_store<atomic_store_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, 171 ro_Xextend32:$extend), 172 GPR32:$val), 173 (STRWroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>; 174 def : Pat<(relaxed_store<atomic_store_32> 175 (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset), GPR32:$val), 176 (STRWui GPR32:$val, GPR64sp:$Rn, uimm12s4:$offset)>; 177 def : Pat<(relaxed_store<atomic_store_32> 178 (am_unscaled32 GPR64sp:$Rn, simm9:$offset), GPR32:$val), 179 (STURWi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>; 180 181 // 64-bit stores 182 def : Pat<(releasing_store<atomic_store_64> GPR64sp:$ptr, GPR64:$val), 183 (STLRX GPR64:$val, GPR64sp:$ptr)>; 184 def : Pat<(relaxed_store<atomic_store_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, 185 ro_Wextend16:$extend), 186 GPR64:$val), 187 (STRXroW GPR64:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>; 188 def : Pat<(relaxed_store<atomic_store_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, 189 ro_Xextend16:$extend), 190 GPR64:$val), 191 (STRXroX GPR64:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>; 192 def : Pat<(relaxed_store<atomic_store_64> 193 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset), GPR64:$val), 194 (STRXui GPR64:$val, GPR64sp:$Rn, uimm12s8:$offset)>; 195 def : Pat<(relaxed_store<atomic_store_64> 196 (am_unscaled64 GPR64sp:$Rn, simm9:$offset), GPR64:$val), 197 (STURXi GPR64:$val, GPR64sp:$Rn, simm9:$offset)>; 198 199 //===---------------------------------- 200 // Low-level exclusive operations 201 //===---------------------------------- 202 203 // Load-exclusives. 204 205 def ldxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 206 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 207 }]>; 208 209 def ldxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 210 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 211 }]>; 212 213 def ldxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 214 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 215 }]>; 216 217 def ldxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 218 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 219 }]>; 220 221 def : Pat<(ldxr_1 GPR64sp:$addr), 222 (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>; 223 def : Pat<(ldxr_2 GPR64sp:$addr), 224 (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>; 225 def : Pat<(ldxr_4 GPR64sp:$addr), 226 (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>; 227 def : Pat<(ldxr_8 GPR64sp:$addr), (LDXRX GPR64sp:$addr)>; 228 229 def : Pat<(and (ldxr_1 GPR64sp:$addr), 0xff), 230 (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>; 231 def : Pat<(and (ldxr_2 GPR64sp:$addr), 0xffff), 232 (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>; 233 def : Pat<(and (ldxr_4 GPR64sp:$addr), 0xffffffff), 234 (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>; 235 236 // Load-exclusives. 237 238 def ldaxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 239 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 240 }]>; 241 242 def ldaxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 243 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 244 }]>; 245 246 def ldaxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 247 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 248 }]>; 249 250 def ldaxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 251 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 252 }]>; 253 254 def : Pat<(ldaxr_1 GPR64sp:$addr), 255 (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>; 256 def : Pat<(ldaxr_2 GPR64sp:$addr), 257 (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>; 258 def : Pat<(ldaxr_4 GPR64sp:$addr), 259 (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>; 260 def : Pat<(ldaxr_8 GPR64sp:$addr), (LDAXRX GPR64sp:$addr)>; 261 262 def : Pat<(and (ldaxr_1 GPR64sp:$addr), 0xff), 263 (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>; 264 def : Pat<(and (ldaxr_2 GPR64sp:$addr), 0xffff), 265 (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>; 266 def : Pat<(and (ldaxr_4 GPR64sp:$addr), 0xffffffff), 267 (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>; 268 269 // Store-exclusives. 270 271 def stxr_1 : PatFrag<(ops node:$val, node:$ptr), 272 (int_aarch64_stxr node:$val, node:$ptr), [{ 273 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 274 }]>; 275 276 def stxr_2 : PatFrag<(ops node:$val, node:$ptr), 277 (int_aarch64_stxr node:$val, node:$ptr), [{ 278 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 279 }]>; 280 281 def stxr_4 : PatFrag<(ops node:$val, node:$ptr), 282 (int_aarch64_stxr node:$val, node:$ptr), [{ 283 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 284 }]>; 285 286 def stxr_8 : PatFrag<(ops node:$val, node:$ptr), 287 (int_aarch64_stxr node:$val, node:$ptr), [{ 288 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 289 }]>; 290 291 292 def : Pat<(stxr_1 GPR64:$val, GPR64sp:$addr), 293 (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 294 def : Pat<(stxr_2 GPR64:$val, GPR64sp:$addr), 295 (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 296 def : Pat<(stxr_4 GPR64:$val, GPR64sp:$addr), 297 (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 298 def : Pat<(stxr_8 GPR64:$val, GPR64sp:$addr), 299 (STXRX GPR64:$val, GPR64sp:$addr)>; 300 301 def : Pat<(stxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr), 302 (STXRB GPR32:$val, GPR64sp:$addr)>; 303 def : Pat<(stxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr), 304 (STXRH GPR32:$val, GPR64sp:$addr)>; 305 def : Pat<(stxr_4 (zext GPR32:$val), GPR64sp:$addr), 306 (STXRW GPR32:$val, GPR64sp:$addr)>; 307 308 def : Pat<(stxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr), 309 (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 310 def : Pat<(stxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr), 311 (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 312 def : Pat<(stxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr), 313 (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 314 315 // Store-release-exclusives. 316 317 def stlxr_1 : PatFrag<(ops node:$val, node:$ptr), 318 (int_aarch64_stlxr node:$val, node:$ptr), [{ 319 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 320 }]>; 321 322 def stlxr_2 : PatFrag<(ops node:$val, node:$ptr), 323 (int_aarch64_stlxr node:$val, node:$ptr), [{ 324 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 325 }]>; 326 327 def stlxr_4 : PatFrag<(ops node:$val, node:$ptr), 328 (int_aarch64_stlxr node:$val, node:$ptr), [{ 329 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 330 }]>; 331 332 def stlxr_8 : PatFrag<(ops node:$val, node:$ptr), 333 (int_aarch64_stlxr node:$val, node:$ptr), [{ 334 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 335 }]>; 336 337 338 def : Pat<(stlxr_1 GPR64:$val, GPR64sp:$addr), 339 (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 340 def : Pat<(stlxr_2 GPR64:$val, GPR64sp:$addr), 341 (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 342 def : Pat<(stlxr_4 GPR64:$val, GPR64sp:$addr), 343 (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 344 def : Pat<(stlxr_8 GPR64:$val, GPR64sp:$addr), 345 (STLXRX GPR64:$val, GPR64sp:$addr)>; 346 347 def : Pat<(stlxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr), 348 (STLXRB GPR32:$val, GPR64sp:$addr)>; 349 def : Pat<(stlxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr), 350 (STLXRH GPR32:$val, GPR64sp:$addr)>; 351 def : Pat<(stlxr_4 (zext GPR32:$val), GPR64sp:$addr), 352 (STLXRW GPR32:$val, GPR64sp:$addr)>; 353 354 def : Pat<(stlxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr), 355 (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 356 def : Pat<(stlxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr), 357 (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 358 def : Pat<(stlxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr), 359 (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 360 361 362 // And clear exclusive. 363 364 def : Pat<(int_aarch64_clrex), (CLREX 0xf)>; 365
