1 //===- SIMemoryLegalizer.cpp ----------------------------------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 /// \file 11 /// Memory legalizer - implements memory model. More information can be 12 /// found here: 13 /// http://llvm.org/docs/AMDGPUUsage.html#memory-model 14 // 15 //===----------------------------------------------------------------------===// 16 17 #include "AMDGPU.h" 18 #include "AMDGPUMachineModuleInfo.h" 19 #include "AMDGPUSubtarget.h" 20 #include "SIDefines.h" 21 #include "SIInstrInfo.h" 22 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 23 #include "Utils/AMDGPUBaseInfo.h" 24 #include "llvm/ADT/BitmaskEnum.h" 25 #include "llvm/ADT/None.h" 26 #include "llvm/ADT/Optional.h" 27 #include "llvm/CodeGen/MachineBasicBlock.h" 28 #include "llvm/CodeGen/MachineFunction.h" 29 #include "llvm/CodeGen/MachineFunctionPass.h" 30 #include "llvm/CodeGen/MachineInstrBuilder.h" 31 #include "llvm/CodeGen/MachineMemOperand.h" 32 #include "llvm/CodeGen/MachineModuleInfo.h" 33 #include "llvm/CodeGen/MachineOperand.h" 34 #include "llvm/IR/DebugLoc.h" 35 #include "llvm/IR/DiagnosticInfo.h" 36 #include "llvm/IR/Function.h" 37 #include "llvm/IR/LLVMContext.h" 38 #include "llvm/MC/MCInstrDesc.h" 39 #include "llvm/Pass.h" 40 #include "llvm/Support/AtomicOrdering.h" 41 #include "llvm/Support/MathExtras.h" 42 #include <cassert> 43 #include <list> 44 45 using namespace llvm; 46 using namespace llvm::AMDGPU; 47 48 #define DEBUG_TYPE "si-memory-legalizer" 49 #define PASS_NAME "SI Memory Legalizer" 50 51 namespace { 52 53 LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE(); 54 55 /// Memory operation flags. Can be ORed together. 56 enum class SIMemOp { 57 NONE = 0u, 58 LOAD = 1u << 0, 59 STORE = 1u << 1, 60 LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ STORE) 61 }; 62 63 /// Position to insert a new instruction relative to an existing 64 /// instruction. 65 enum class Position { 66 BEFORE, 67 AFTER 68 }; 69 70 /// The atomic synchronization scopes supported by the AMDGPU target. 71 enum class SIAtomicScope { 72 NONE, 73 SINGLETHREAD, 74 WAVEFRONT, 75 WORKGROUP, 76 AGENT, 77 SYSTEM 78 }; 79 80 /// The distinct address spaces supported by the AMDGPU target for 81 /// atomic memory operation. Can be ORed toether. 82 enum class SIAtomicAddrSpace { 83 NONE = 0u, 84 GLOBAL = 1u << 0, 85 LDS = 1u << 1, 86 SCRATCH = 1u << 2, 87 GDS = 1u << 3, 88 OTHER = 1u << 4, 89 90 /// The address spaces that can be accessed by a FLAT instruction. 91 FLAT = GLOBAL | LDS | SCRATCH, 92 93 /// The address spaces that support atomic instructions. 94 ATOMIC = GLOBAL | LDS | SCRATCH | GDS, 95 96 /// All address spaces. 97 ALL = GLOBAL | LDS | SCRATCH | GDS | OTHER, 98 99 LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ ALL) 100 }; 101 102 /// Sets named bit \p BitName to "true" if present in instruction \p MI. 103 /// \returns Returns true if \p MI is modified, false otherwise. 104 template <uint16_t BitName> 105 bool enableNamedBit(const MachineBasicBlock::iterator &MI) { 106 int BitIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), BitName); 107 if (BitIdx == -1) 108 return false; 109 110 MachineOperand &Bit = MI->getOperand(BitIdx); 111 if (Bit.getImm() != 0) 112 return false; 113 114 Bit.setImm(1); 115 return true; 116 } 117 118 class SIMemOpInfo final { 119 private: 120 121 friend class SIMemOpAccess; 122 123 AtomicOrdering Ordering = AtomicOrdering::NotAtomic; 124 AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic; 125 SIAtomicScope Scope = SIAtomicScope::SYSTEM; 126 SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::NONE; 127 SIAtomicAddrSpace InstrAddrSpace = SIAtomicAddrSpace::NONE; 128 bool IsCrossAddressSpaceOrdering = false; 129 bool IsNonTemporal = false; 130 131 SIMemOpInfo(AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent, 132 SIAtomicScope Scope = SIAtomicScope::SYSTEM, 133 SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::ATOMIC, 134 SIAtomicAddrSpace InstrAddrSpace = SIAtomicAddrSpace::ALL, 135 bool IsCrossAddressSpaceOrdering = true, 136 AtomicOrdering FailureOrdering = 137 AtomicOrdering::SequentiallyConsistent, 138 bool IsNonTemporal = false) 139 : Ordering(Ordering), FailureOrdering(FailureOrdering), 140 Scope(Scope), OrderingAddrSpace(OrderingAddrSpace), 141 InstrAddrSpace(InstrAddrSpace), 142 IsCrossAddressSpaceOrdering(IsCrossAddressSpaceOrdering), 143 IsNonTemporal(IsNonTemporal) { 144 // There is also no cross address space ordering if the ordering 145 // address space is the same as the instruction address space and 146 // only contains a single address space. 147 if ((OrderingAddrSpace == InstrAddrSpace) && 148 isPowerOf2_32(uint32_t(InstrAddrSpace))) 149 IsCrossAddressSpaceOrdering = false; 150 } 151 152 public: 153 /// \returns Atomic synchronization scope of the machine instruction used to 154 /// create this SIMemOpInfo. 155 SIAtomicScope getScope() const { 156 return Scope; 157 } 158 159 /// \returns Ordering constraint of the machine instruction used to 160 /// create this SIMemOpInfo. 161 AtomicOrdering getOrdering() const { 162 return Ordering; 163 } 164 165 /// \returns Failure ordering constraint of the machine instruction used to 166 /// create this SIMemOpInfo. 167 AtomicOrdering getFailureOrdering() const { 168 return FailureOrdering; 169 } 170 171 /// \returns The address spaces be accessed by the machine 172 /// instruction used to create this SiMemOpInfo. 173 SIAtomicAddrSpace getInstrAddrSpace() const { 174 return InstrAddrSpace; 175 } 176 177 /// \returns The address spaces that must be ordered by the machine 178 /// instruction used to create this SiMemOpInfo. 179 SIAtomicAddrSpace getOrderingAddrSpace() const { 180 return OrderingAddrSpace; 181 } 182 183 /// \returns Return true iff memory ordering of operations on 184 /// different address spaces is required. 185 bool getIsCrossAddressSpaceOrdering() const { 186 return IsCrossAddressSpaceOrdering; 187 } 188 189 /// \returns True if memory access of the machine instruction used to 190 /// create this SIMemOpInfo is non-temporal, false otherwise. 191 bool isNonTemporal() const { 192 return IsNonTemporal; 193 } 194 195 /// \returns True if ordering constraint of the machine instruction used to 196 /// create this SIMemOpInfo is unordered or higher, false otherwise. 197 bool isAtomic() const { 198 return Ordering != AtomicOrdering::NotAtomic; 199 } 200 201 }; 202 203 class SIMemOpAccess final { 204 private: 205 206 AMDGPUAS SIAddrSpaceInfo; 207 AMDGPUMachineModuleInfo *MMI = nullptr; 208 209 /// Reports unsupported message \p Msg for \p MI to LLVM context. 210 void reportUnsupported(const MachineBasicBlock::iterator &MI, 211 const char *Msg) const; 212 213 /// Inspects the target synchonization scope \p SSID and determines 214 /// the SI atomic scope it corresponds to, the address spaces it 215 /// covers, and whether the memory ordering applies between address 216 /// spaces. 217 Optional<std::tuple<SIAtomicScope, SIAtomicAddrSpace, bool>> 218 toSIAtomicScope(SyncScope::ID SSID, SIAtomicAddrSpace InstrScope) const; 219 220 /// \return Return a bit set of the address spaces accessed by \p AS. 221 SIAtomicAddrSpace toSIAtomicAddrSpace(unsigned AS) const; 222 223 /// \returns Info constructed from \p MI, which has at least machine memory 224 /// operand. 225 Optional<SIMemOpInfo> constructFromMIWithMMO( 226 const MachineBasicBlock::iterator &MI) const; 227 228 public: 229 /// Construct class to support accessing the machine memory operands 230 /// of instructions in the machine function \p MF. 231 SIMemOpAccess(MachineFunction &MF); 232 233 /// \returns Load info if \p MI is a load operation, "None" otherwise. 234 Optional<SIMemOpInfo> getLoadInfo( 235 const MachineBasicBlock::iterator &MI) const; 236 237 /// \returns Store info if \p MI is a store operation, "None" otherwise. 238 Optional<SIMemOpInfo> getStoreInfo( 239 const MachineBasicBlock::iterator &MI) const; 240 241 /// \returns Atomic fence info if \p MI is an atomic fence operation, 242 /// "None" otherwise. 243 Optional<SIMemOpInfo> getAtomicFenceInfo( 244 const MachineBasicBlock::iterator &MI) const; 245 246 /// \returns Atomic cmpxchg/rmw info if \p MI is an atomic cmpxchg or 247 /// rmw operation, "None" otherwise. 248 Optional<SIMemOpInfo> getAtomicCmpxchgOrRmwInfo( 249 const MachineBasicBlock::iterator &MI) const; 250 }; 251 252 class SICacheControl { 253 protected: 254 255 /// Instruction info. 256 const SIInstrInfo *TII = nullptr; 257 258 IsaInfo::IsaVersion IV; 259 260 SICacheControl(const GCNSubtarget &ST); 261 262 public: 263 264 /// Create a cache control for the subtarget \p ST. 265 static std::unique_ptr<SICacheControl> create(const GCNSubtarget &ST); 266 267 /// Update \p MI memory load instruction to bypass any caches up to 268 /// the \p Scope memory scope for address spaces \p 269 /// AddrSpace. Return true iff the instruction was modified. 270 virtual bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI, 271 SIAtomicScope Scope, 272 SIAtomicAddrSpace AddrSpace) const = 0; 273 274 /// Update \p MI memory instruction to indicate it is 275 /// nontemporal. Return true iff the instruction was modified. 276 virtual bool enableNonTemporal(const MachineBasicBlock::iterator &MI) 277 const = 0; 278 279 /// Inserts any necessary instructions at position \p Pos relative 280 /// to instruction \p MI to ensure any caches associated with 281 /// address spaces \p AddrSpace for memory scopes up to memory scope 282 /// \p Scope are invalidated. Returns true iff any instructions 283 /// inserted. 284 virtual bool insertCacheInvalidate(MachineBasicBlock::iterator &MI, 285 SIAtomicScope Scope, 286 SIAtomicAddrSpace AddrSpace, 287 Position Pos) const = 0; 288 289 /// Inserts any necessary instructions at position \p Pos relative 290 /// to instruction \p MI to ensure memory instructions of kind \p Op 291 /// associated with address spaces \p AddrSpace have completed as 292 /// observed by other memory instructions executing in memory scope 293 /// \p Scope. \p IsCrossAddrSpaceOrdering indicates if the memory 294 /// ordering is between address spaces. Returns true iff any 295 /// instructions inserted. 296 virtual bool insertWait(MachineBasicBlock::iterator &MI, 297 SIAtomicScope Scope, 298 SIAtomicAddrSpace AddrSpace, 299 SIMemOp Op, 300 bool IsCrossAddrSpaceOrdering, 301 Position Pos) const = 0; 302 303 /// Virtual destructor to allow derivations to be deleted. 304 virtual ~SICacheControl() = default; 305 306 }; 307 308 class SIGfx6CacheControl : public SICacheControl { 309 protected: 310 311 /// Sets GLC bit to "true" if present in \p MI. Returns true if \p MI 312 /// is modified, false otherwise. 313 bool enableGLCBit(const MachineBasicBlock::iterator &MI) const { 314 return enableNamedBit<AMDGPU::OpName::glc>(MI); 315 } 316 317 /// Sets SLC bit to "true" if present in \p MI. Returns true if \p MI 318 /// is modified, false otherwise. 319 bool enableSLCBit(const MachineBasicBlock::iterator &MI) const { 320 return enableNamedBit<AMDGPU::OpName::slc>(MI); 321 } 322 323 public: 324 325 SIGfx6CacheControl(const GCNSubtarget &ST) : SICacheControl(ST) {}; 326 327 bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI, 328 SIAtomicScope Scope, 329 SIAtomicAddrSpace AddrSpace) const override; 330 331 bool enableNonTemporal(const MachineBasicBlock::iterator &MI) const override; 332 333 bool insertCacheInvalidate(MachineBasicBlock::iterator &MI, 334 SIAtomicScope Scope, 335 SIAtomicAddrSpace AddrSpace, 336 Position Pos) const override; 337 338 bool insertWait(MachineBasicBlock::iterator &MI, 339 SIAtomicScope Scope, 340 SIAtomicAddrSpace AddrSpace, 341 SIMemOp Op, 342 bool IsCrossAddrSpaceOrdering, 343 Position Pos) const override; 344 }; 345 346 class SIGfx7CacheControl : public SIGfx6CacheControl { 347 public: 348 349 SIGfx7CacheControl(const GCNSubtarget &ST) : SIGfx6CacheControl(ST) {}; 350 351 bool insertCacheInvalidate(MachineBasicBlock::iterator &MI, 352 SIAtomicScope Scope, 353 SIAtomicAddrSpace AddrSpace, 354 Position Pos) const override; 355 356 }; 357 358 class SIMemoryLegalizer final : public MachineFunctionPass { 359 private: 360 361 /// Cache Control. 362 std::unique_ptr<SICacheControl> CC = nullptr; 363 364 /// List of atomic pseudo instructions. 365 std::list<MachineBasicBlock::iterator> AtomicPseudoMIs; 366 367 /// Return true iff instruction \p MI is a atomic instruction that 368 /// returns a result. 369 bool isAtomicRet(const MachineInstr &MI) const { 370 return AMDGPU::getAtomicNoRetOp(MI.getOpcode()) != -1; 371 } 372 373 /// Removes all processed atomic pseudo instructions from the current 374 /// function. Returns true if current function is modified, false otherwise. 375 bool removeAtomicPseudoMIs(); 376 377 /// Expands load operation \p MI. Returns true if instructions are 378 /// added/deleted or \p MI is modified, false otherwise. 379 bool expandLoad(const SIMemOpInfo &MOI, 380 MachineBasicBlock::iterator &MI); 381 /// Expands store operation \p MI. Returns true if instructions are 382 /// added/deleted or \p MI is modified, false otherwise. 383 bool expandStore(const SIMemOpInfo &MOI, 384 MachineBasicBlock::iterator &MI); 385 /// Expands atomic fence operation \p MI. Returns true if 386 /// instructions are added/deleted or \p MI is modified, false otherwise. 387 bool expandAtomicFence(const SIMemOpInfo &MOI, 388 MachineBasicBlock::iterator &MI); 389 /// Expands atomic cmpxchg or rmw operation \p MI. Returns true if 390 /// instructions are added/deleted or \p MI is modified, false otherwise. 391 bool expandAtomicCmpxchgOrRmw(const SIMemOpInfo &MOI, 392 MachineBasicBlock::iterator &MI); 393 394 public: 395 static char ID; 396 397 SIMemoryLegalizer() : MachineFunctionPass(ID) {} 398 399 void getAnalysisUsage(AnalysisUsage &AU) const override { 400 AU.setPreservesCFG(); 401 MachineFunctionPass::getAnalysisUsage(AU); 402 } 403 404 StringRef getPassName() const override { 405 return PASS_NAME; 406 } 407 408 bool runOnMachineFunction(MachineFunction &MF) override; 409 }; 410 411 } // end namespace anonymous 412 413 void SIMemOpAccess::reportUnsupported(const MachineBasicBlock::iterator &MI, 414 const char *Msg) const { 415 const Function &Func = MI->getParent()->getParent()->getFunction(); 416 DiagnosticInfoUnsupported Diag(Func, Msg, MI->getDebugLoc()); 417 Func.getContext().diagnose(Diag); 418 } 419 420 Optional<std::tuple<SIAtomicScope, SIAtomicAddrSpace, bool>> 421 SIMemOpAccess::toSIAtomicScope(SyncScope::ID SSID, 422 SIAtomicAddrSpace InstrScope) const { 423 /// TODO: For now assume OpenCL memory model which treats each 424 /// address space as having a separate happens-before relation, and 425 /// so an instruction only has ordering with respect to the address 426 /// space it accesses, and if it accesses multiple address spaces it 427 /// does not require ordering of operations in different address 428 /// spaces. 429 if (SSID == SyncScope::System) 430 return std::make_tuple(SIAtomicScope::SYSTEM, 431 SIAtomicAddrSpace::ATOMIC & InstrScope, 432 false); 433 if (SSID == MMI->getAgentSSID()) 434 return std::make_tuple(SIAtomicScope::AGENT, 435 SIAtomicAddrSpace::ATOMIC & InstrScope, 436 false); 437 if (SSID == MMI->getWorkgroupSSID()) 438 return std::make_tuple(SIAtomicScope::WORKGROUP, 439 SIAtomicAddrSpace::ATOMIC & InstrScope, 440 false); 441 if (SSID == MMI->getWavefrontSSID()) 442 return std::make_tuple(SIAtomicScope::WAVEFRONT, 443 SIAtomicAddrSpace::ATOMIC & InstrScope, 444 false); 445 if (SSID == SyncScope::SingleThread) 446 return std::make_tuple(SIAtomicScope::SINGLETHREAD, 447 SIAtomicAddrSpace::ATOMIC & InstrScope, 448 false); 449 /// TODO: To support HSA Memory Model need to add additional memory 450 /// scopes that specify that do require cross address space 451 /// ordering. 452 return None; 453 } 454 455 SIAtomicAddrSpace SIMemOpAccess::toSIAtomicAddrSpace(unsigned AS) const { 456 if (AS == SIAddrSpaceInfo.FLAT_ADDRESS) 457 return SIAtomicAddrSpace::FLAT; 458 if (AS == SIAddrSpaceInfo.GLOBAL_ADDRESS) 459 return SIAtomicAddrSpace::GLOBAL; 460 if (AS == SIAddrSpaceInfo.LOCAL_ADDRESS) 461 return SIAtomicAddrSpace::LDS; 462 if (AS == SIAddrSpaceInfo.PRIVATE_ADDRESS) 463 return SIAtomicAddrSpace::SCRATCH; 464 if (AS == SIAddrSpaceInfo.REGION_ADDRESS) 465 return SIAtomicAddrSpace::GDS; 466 467 return SIAtomicAddrSpace::OTHER; 468 } 469 470 SIMemOpAccess::SIMemOpAccess(MachineFunction &MF) { 471 SIAddrSpaceInfo = getAMDGPUAS(MF.getTarget()); 472 MMI = &MF.getMMI().getObjFileInfo<AMDGPUMachineModuleInfo>(); 473 } 474 475 Optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO( 476 const MachineBasicBlock::iterator &MI) const { 477 assert(MI->getNumMemOperands() > 0); 478 479 SyncScope::ID SSID = SyncScope::SingleThread; 480 AtomicOrdering Ordering = AtomicOrdering::NotAtomic; 481 AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic; 482 SIAtomicAddrSpace InstrAddrSpace = SIAtomicAddrSpace::NONE; 483 bool IsNonTemporal = true; 484 485 // Validator should check whether or not MMOs cover the entire set of 486 // locations accessed by the memory instruction. 487 for (const auto &MMO : MI->memoperands()) { 488 IsNonTemporal &= MMO->isNonTemporal(); 489 InstrAddrSpace |= 490 toSIAtomicAddrSpace(MMO->getPointerInfo().getAddrSpace()); 491 AtomicOrdering OpOrdering = MMO->getOrdering(); 492 if (OpOrdering != AtomicOrdering::NotAtomic) { 493 const auto &IsSyncScopeInclusion = 494 MMI->isSyncScopeInclusion(SSID, MMO->getSyncScopeID()); 495 if (!IsSyncScopeInclusion) { 496 reportUnsupported(MI, 497 "Unsupported non-inclusive atomic synchronization scope"); 498 return None; 499 } 500 501 SSID = IsSyncScopeInclusion.getValue() ? SSID : MMO->getSyncScopeID(); 502 Ordering = 503 isStrongerThan(Ordering, OpOrdering) ? 504 Ordering : MMO->getOrdering(); 505 assert(MMO->getFailureOrdering() != AtomicOrdering::Release && 506 MMO->getFailureOrdering() != AtomicOrdering::AcquireRelease); 507 FailureOrdering = 508 isStrongerThan(FailureOrdering, MMO->getFailureOrdering()) ? 509 FailureOrdering : MMO->getFailureOrdering(); 510 } 511 } 512 513 SIAtomicScope Scope = SIAtomicScope::NONE; 514 SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::NONE; 515 bool IsCrossAddressSpaceOrdering = false; 516 if (Ordering != AtomicOrdering::NotAtomic) { 517 auto ScopeOrNone = toSIAtomicScope(SSID, InstrAddrSpace); 518 if (!ScopeOrNone) { 519 reportUnsupported(MI, "Unsupported atomic synchronization scope"); 520 return None; 521 } 522 std::tie(Scope, OrderingAddrSpace, IsCrossAddressSpaceOrdering) = 523 ScopeOrNone.getValue(); 524 if ((OrderingAddrSpace == SIAtomicAddrSpace::NONE) || 525 ((OrderingAddrSpace & SIAtomicAddrSpace::ATOMIC) != OrderingAddrSpace)) { 526 reportUnsupported(MI, "Unsupported atomic address space"); 527 return None; 528 } 529 } 530 return SIMemOpInfo(Ordering, Scope, OrderingAddrSpace, InstrAddrSpace, 531 IsCrossAddressSpaceOrdering, FailureOrdering, IsNonTemporal); 532 } 533 534 Optional<SIMemOpInfo> SIMemOpAccess::getLoadInfo( 535 const MachineBasicBlock::iterator &MI) const { 536 assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); 537 538 if (!(MI->mayLoad() && !MI->mayStore())) 539 return None; 540 541 // Be conservative if there are no memory operands. 542 if (MI->getNumMemOperands() == 0) 543 return SIMemOpInfo(); 544 545 return constructFromMIWithMMO(MI); 546 } 547 548 Optional<SIMemOpInfo> SIMemOpAccess::getStoreInfo( 549 const MachineBasicBlock::iterator &MI) const { 550 assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); 551 552 if (!(!MI->mayLoad() && MI->mayStore())) 553 return None; 554 555 // Be conservative if there are no memory operands. 556 if (MI->getNumMemOperands() == 0) 557 return SIMemOpInfo(); 558 559 return constructFromMIWithMMO(MI); 560 } 561 562 Optional<SIMemOpInfo> SIMemOpAccess::getAtomicFenceInfo( 563 const MachineBasicBlock::iterator &MI) const { 564 assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); 565 566 if (MI->getOpcode() != AMDGPU::ATOMIC_FENCE) 567 return None; 568 569 AtomicOrdering Ordering = 570 static_cast<AtomicOrdering>(MI->getOperand(0).getImm()); 571 572 SyncScope::ID SSID = static_cast<SyncScope::ID>(MI->getOperand(1).getImm()); 573 auto ScopeOrNone = toSIAtomicScope(SSID, SIAtomicAddrSpace::ATOMIC); 574 if (!ScopeOrNone) { 575 reportUnsupported(MI, "Unsupported atomic synchronization scope"); 576 return None; 577 } 578 579 SIAtomicScope Scope = SIAtomicScope::NONE; 580 SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::NONE; 581 bool IsCrossAddressSpaceOrdering = false; 582 std::tie(Scope, OrderingAddrSpace, IsCrossAddressSpaceOrdering) = 583 ScopeOrNone.getValue(); 584 585 if ((OrderingAddrSpace == SIAtomicAddrSpace::NONE) || 586 ((OrderingAddrSpace & SIAtomicAddrSpace::ATOMIC) != OrderingAddrSpace)) { 587 reportUnsupported(MI, "Unsupported atomic address space"); 588 return None; 589 } 590 591 return SIMemOpInfo(Ordering, Scope, OrderingAddrSpace, SIAtomicAddrSpace::ATOMIC, 592 IsCrossAddressSpaceOrdering); 593 } 594 595 Optional<SIMemOpInfo> SIMemOpAccess::getAtomicCmpxchgOrRmwInfo( 596 const MachineBasicBlock::iterator &MI) const { 597 assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); 598 599 if (!(MI->mayLoad() && MI->mayStore())) 600 return None; 601 602 // Be conservative if there are no memory operands. 603 if (MI->getNumMemOperands() == 0) 604 return SIMemOpInfo(); 605 606 return constructFromMIWithMMO(MI); 607 } 608 609 SICacheControl::SICacheControl(const GCNSubtarget &ST) { 610 TII = ST.getInstrInfo(); 611 IV = IsaInfo::getIsaVersion(ST.getFeatureBits()); 612 } 613 614 /* static */ 615 std::unique_ptr<SICacheControl> SICacheControl::create(const GCNSubtarget &ST) { 616 GCNSubtarget::Generation Generation = ST.getGeneration(); 617 if (Generation <= AMDGPUSubtarget::SOUTHERN_ISLANDS) 618 return make_unique<SIGfx6CacheControl>(ST); 619 return make_unique<SIGfx7CacheControl>(ST); 620 } 621 622 bool SIGfx6CacheControl::enableLoadCacheBypass( 623 const MachineBasicBlock::iterator &MI, 624 SIAtomicScope Scope, 625 SIAtomicAddrSpace AddrSpace) const { 626 assert(MI->mayLoad() && !MI->mayStore()); 627 bool Changed = false; 628 629 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) { 630 /// TODO: Do not set glc for rmw atomic operations as they 631 /// implicitly bypass the L1 cache. 632 633 switch (Scope) { 634 case SIAtomicScope::SYSTEM: 635 case SIAtomicScope::AGENT: 636 Changed |= enableGLCBit(MI); 637 break; 638 case SIAtomicScope::WORKGROUP: 639 case SIAtomicScope::WAVEFRONT: 640 case SIAtomicScope::SINGLETHREAD: 641 // No cache to bypass. 642 break; 643 default: 644 llvm_unreachable("Unsupported synchronization scope"); 645 } 646 } 647 648 /// The scratch address space does not need the global memory caches 649 /// to be bypassed as all memory operations by the same thread are 650 /// sequentially consistent, and no other thread can access scratch 651 /// memory. 652 653 /// Other address spaces do not hava a cache. 654 655 return Changed; 656 } 657 658 bool SIGfx6CacheControl::enableNonTemporal( 659 const MachineBasicBlock::iterator &MI) const { 660 assert(MI->mayLoad() ^ MI->mayStore()); 661 bool Changed = false; 662 663 /// TODO: Do not enableGLCBit if rmw atomic. 664 Changed |= enableGLCBit(MI); 665 Changed |= enableSLCBit(MI); 666 667 return Changed; 668 } 669 670 bool SIGfx6CacheControl::insertCacheInvalidate(MachineBasicBlock::iterator &MI, 671 SIAtomicScope Scope, 672 SIAtomicAddrSpace AddrSpace, 673 Position Pos) const { 674 bool Changed = false; 675 676 MachineBasicBlock &MBB = *MI->getParent(); 677 DebugLoc DL = MI->getDebugLoc(); 678 679 if (Pos == Position::AFTER) 680 ++MI; 681 682 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) { 683 switch (Scope) { 684 case SIAtomicScope::SYSTEM: 685 case SIAtomicScope::AGENT: 686 BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_WBINVL1)); 687 Changed = true; 688 break; 689 case SIAtomicScope::WORKGROUP: 690 case SIAtomicScope::WAVEFRONT: 691 case SIAtomicScope::SINGLETHREAD: 692 // No cache to invalidate. 693 break; 694 default: 695 llvm_unreachable("Unsupported synchronization scope"); 696 } 697 } 698 699 /// The scratch address space does not need the global memory cache 700 /// to be flushed as all memory operations by the same thread are 701 /// sequentially consistent, and no other thread can access scratch 702 /// memory. 703 704 /// Other address spaces do not hava a cache. 705 706 if (Pos == Position::AFTER) 707 --MI; 708 709 return Changed; 710 } 711 712 bool SIGfx6CacheControl::insertWait(MachineBasicBlock::iterator &MI, 713 SIAtomicScope Scope, 714 SIAtomicAddrSpace AddrSpace, 715 SIMemOp Op, 716 bool IsCrossAddrSpaceOrdering, 717 Position Pos) const { 718 bool Changed = false; 719 720 MachineBasicBlock &MBB = *MI->getParent(); 721 DebugLoc DL = MI->getDebugLoc(); 722 723 if (Pos == Position::AFTER) 724 ++MI; 725 726 bool VMCnt = false; 727 bool LGKMCnt = false; 728 bool EXPCnt = false; 729 730 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) { 731 switch (Scope) { 732 case SIAtomicScope::SYSTEM: 733 case SIAtomicScope::AGENT: 734 VMCnt = true; 735 break; 736 case SIAtomicScope::WORKGROUP: 737 case SIAtomicScope::WAVEFRONT: 738 case SIAtomicScope::SINGLETHREAD: 739 // The L1 cache keeps all memory operations in order for 740 // wavefronts in the same work-group. 741 break; 742 default: 743 llvm_unreachable("Unsupported synchronization scope"); 744 } 745 } 746 747 if ((AddrSpace & SIAtomicAddrSpace::LDS) != SIAtomicAddrSpace::NONE) { 748 switch (Scope) { 749 case SIAtomicScope::SYSTEM: 750 case SIAtomicScope::AGENT: 751 case SIAtomicScope::WORKGROUP: 752 // If no cross address space ordering then an LDS waitcnt is not 753 // needed as LDS operations for all waves are executed in a 754 // total global ordering as observed by all waves. Required if 755 // also synchronizing with global/GDS memory as LDS operations 756 // could be reordered with respect to later global/GDS memory 757 // operations of the same wave. 758 LGKMCnt = IsCrossAddrSpaceOrdering; 759 break; 760 case SIAtomicScope::WAVEFRONT: 761 case SIAtomicScope::SINGLETHREAD: 762 // The LDS keeps all memory operations in order for 763 // the same wavesfront. 764 break; 765 default: 766 llvm_unreachable("Unsupported synchronization scope"); 767 } 768 } 769 770 if ((AddrSpace & SIAtomicAddrSpace::GDS) != SIAtomicAddrSpace::NONE) { 771 switch (Scope) { 772 case SIAtomicScope::SYSTEM: 773 case SIAtomicScope::AGENT: 774 // If no cross address space ordering then an GDS waitcnt is not 775 // needed as GDS operations for all waves are executed in a 776 // total global ordering as observed by all waves. Required if 777 // also synchronizing with global/LDS memory as GDS operations 778 // could be reordered with respect to later global/LDS memory 779 // operations of the same wave. 780 EXPCnt = IsCrossAddrSpaceOrdering; 781 break; 782 case SIAtomicScope::WORKGROUP: 783 case SIAtomicScope::WAVEFRONT: 784 case SIAtomicScope::SINGLETHREAD: 785 // The GDS keeps all memory operations in order for 786 // the same work-group. 787 break; 788 default: 789 llvm_unreachable("Unsupported synchronization scope"); 790 } 791 } 792 793 if (VMCnt || LGKMCnt || EXPCnt) { 794 unsigned WaitCntImmediate = 795 AMDGPU::encodeWaitcnt(IV, 796 VMCnt ? 0 : getVmcntBitMask(IV), 797 EXPCnt ? 0 : getExpcntBitMask(IV), 798 LGKMCnt ? 0 : getLgkmcntBitMask(IV)); 799 BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAITCNT)).addImm(WaitCntImmediate); 800 Changed = true; 801 } 802 803 if (Pos == Position::AFTER) 804 --MI; 805 806 return Changed; 807 } 808 809 bool SIGfx7CacheControl::insertCacheInvalidate(MachineBasicBlock::iterator &MI, 810 SIAtomicScope Scope, 811 SIAtomicAddrSpace AddrSpace, 812 Position Pos) const { 813 bool Changed = false; 814 815 MachineBasicBlock &MBB = *MI->getParent(); 816 DebugLoc DL = MI->getDebugLoc(); 817 818 if (Pos == Position::AFTER) 819 ++MI; 820 821 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) { 822 switch (Scope) { 823 case SIAtomicScope::SYSTEM: 824 case SIAtomicScope::AGENT: 825 BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_WBINVL1_VOL)); 826 Changed = true; 827 break; 828 case SIAtomicScope::WORKGROUP: 829 case SIAtomicScope::WAVEFRONT: 830 case SIAtomicScope::SINGLETHREAD: 831 // No cache to invalidate. 832 break; 833 default: 834 llvm_unreachable("Unsupported synchronization scope"); 835 } 836 } 837 838 /// The scratch address space does not need the global memory cache 839 /// to be flushed as all memory operations by the same thread are 840 /// sequentially consistent, and no other thread can access scratch 841 /// memory. 842 843 /// Other address spaces do not hava a cache. 844 845 if (Pos == Position::AFTER) 846 --MI; 847 848 return Changed; 849 } 850 851 bool SIMemoryLegalizer::removeAtomicPseudoMIs() { 852 if (AtomicPseudoMIs.empty()) 853 return false; 854 855 for (auto &MI : AtomicPseudoMIs) 856 MI->eraseFromParent(); 857 858 AtomicPseudoMIs.clear(); 859 return true; 860 } 861 862 bool SIMemoryLegalizer::expandLoad(const SIMemOpInfo &MOI, 863 MachineBasicBlock::iterator &MI) { 864 assert(MI->mayLoad() && !MI->mayStore()); 865 866 bool Changed = false; 867 868 if (MOI.isAtomic()) { 869 if (MOI.getOrdering() == AtomicOrdering::Monotonic || 870 MOI.getOrdering() == AtomicOrdering::Acquire || 871 MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) { 872 Changed |= CC->enableLoadCacheBypass(MI, MOI.getScope(), 873 MOI.getOrderingAddrSpace()); 874 } 875 876 if (MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) 877 Changed |= CC->insertWait(MI, MOI.getScope(), 878 MOI.getOrderingAddrSpace(), 879 SIMemOp::LOAD | SIMemOp::STORE, 880 MOI.getIsCrossAddressSpaceOrdering(), 881 Position::BEFORE); 882 883 if (MOI.getOrdering() == AtomicOrdering::Acquire || 884 MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) { 885 Changed |= CC->insertWait(MI, MOI.getScope(), 886 MOI.getInstrAddrSpace(), 887 SIMemOp::LOAD, 888 MOI.getIsCrossAddressSpaceOrdering(), 889 Position::AFTER); 890 Changed |= CC->insertCacheInvalidate(MI, MOI.getScope(), 891 MOI.getOrderingAddrSpace(), 892 Position::AFTER); 893 } 894 895 return Changed; 896 } 897 898 // Atomic instructions do not have the nontemporal attribute. 899 if (MOI.isNonTemporal()) { 900 Changed |= CC->enableNonTemporal(MI); 901 return Changed; 902 } 903 904 return Changed; 905 } 906 907 bool SIMemoryLegalizer::expandStore(const SIMemOpInfo &MOI, 908 MachineBasicBlock::iterator &MI) { 909 assert(!MI->mayLoad() && MI->mayStore()); 910 911 bool Changed = false; 912 913 if (MOI.isAtomic()) { 914 if (MOI.getOrdering() == AtomicOrdering::Release || 915 MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) 916 Changed |= CC->insertWait(MI, MOI.getScope(), 917 MOI.getOrderingAddrSpace(), 918 SIMemOp::LOAD | SIMemOp::STORE, 919 MOI.getIsCrossAddressSpaceOrdering(), 920 Position::BEFORE); 921 922 return Changed; 923 } 924 925 // Atomic instructions do not have the nontemporal attribute. 926 if (MOI.isNonTemporal()) { 927 Changed |= CC->enableNonTemporal(MI); 928 return Changed; 929 } 930 931 return Changed; 932 } 933 934 bool SIMemoryLegalizer::expandAtomicFence(const SIMemOpInfo &MOI, 935 MachineBasicBlock::iterator &MI) { 936 assert(MI->getOpcode() == AMDGPU::ATOMIC_FENCE); 937 938 AtomicPseudoMIs.push_back(MI); 939 bool Changed = false; 940 941 if (MOI.isAtomic()) { 942 if (MOI.getOrdering() == AtomicOrdering::Acquire || 943 MOI.getOrdering() == AtomicOrdering::Release || 944 MOI.getOrdering() == AtomicOrdering::AcquireRelease || 945 MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) 946 /// TODO: This relies on a barrier always generating a waitcnt 947 /// for LDS to ensure it is not reordered with the completion of 948 /// the proceeding LDS operations. If barrier had a memory 949 /// ordering and memory scope, then library does not need to 950 /// generate a fence. Could add support in this file for 951 /// barrier. SIInsertWaitcnt.cpp could then stop unconditionally 952 /// adding waitcnt before a S_BARRIER. 953 Changed |= CC->insertWait(MI, MOI.getScope(), 954 MOI.getOrderingAddrSpace(), 955 SIMemOp::LOAD | SIMemOp::STORE, 956 MOI.getIsCrossAddressSpaceOrdering(), 957 Position::BEFORE); 958 959 if (MOI.getOrdering() == AtomicOrdering::Acquire || 960 MOI.getOrdering() == AtomicOrdering::AcquireRelease || 961 MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) 962 Changed |= CC->insertCacheInvalidate(MI, MOI.getScope(), 963 MOI.getOrderingAddrSpace(), 964 Position::BEFORE); 965 966 return Changed; 967 } 968 969 return Changed; 970 } 971 972 bool SIMemoryLegalizer::expandAtomicCmpxchgOrRmw(const SIMemOpInfo &MOI, 973 MachineBasicBlock::iterator &MI) { 974 assert(MI->mayLoad() && MI->mayStore()); 975 976 bool Changed = false; 977 978 if (MOI.isAtomic()) { 979 if (MOI.getOrdering() == AtomicOrdering::Release || 980 MOI.getOrdering() == AtomicOrdering::AcquireRelease || 981 MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent || 982 MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent) 983 Changed |= CC->insertWait(MI, MOI.getScope(), 984 MOI.getOrderingAddrSpace(), 985 SIMemOp::LOAD | SIMemOp::STORE, 986 MOI.getIsCrossAddressSpaceOrdering(), 987 Position::BEFORE); 988 989 if (MOI.getOrdering() == AtomicOrdering::Acquire || 990 MOI.getOrdering() == AtomicOrdering::AcquireRelease || 991 MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent || 992 MOI.getFailureOrdering() == AtomicOrdering::Acquire || 993 MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent) { 994 Changed |= CC->insertWait(MI, MOI.getScope(), 995 MOI.getOrderingAddrSpace(), 996 isAtomicRet(*MI) ? SIMemOp::LOAD : 997 SIMemOp::STORE, 998 MOI.getIsCrossAddressSpaceOrdering(), 999 Position::AFTER); 1000 Changed |= CC->insertCacheInvalidate(MI, MOI.getScope(), 1001 MOI.getOrderingAddrSpace(), 1002 Position::AFTER); 1003 } 1004 1005 return Changed; 1006 } 1007 1008 return Changed; 1009 } 1010 1011 bool SIMemoryLegalizer::runOnMachineFunction(MachineFunction &MF) { 1012 bool Changed = false; 1013 1014 SIMemOpAccess MOA(MF); 1015 CC = SICacheControl::create(MF.getSubtarget<GCNSubtarget>()); 1016 1017 for (auto &MBB : MF) { 1018 for (auto MI = MBB.begin(); MI != MBB.end(); ++MI) { 1019 if (!(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic)) 1020 continue; 1021 1022 if (const auto &MOI = MOA.getLoadInfo(MI)) 1023 Changed |= expandLoad(MOI.getValue(), MI); 1024 else if (const auto &MOI = MOA.getStoreInfo(MI)) 1025 Changed |= expandStore(MOI.getValue(), MI); 1026 else if (const auto &MOI = MOA.getAtomicFenceInfo(MI)) 1027 Changed |= expandAtomicFence(MOI.getValue(), MI); 1028 else if (const auto &MOI = MOA.getAtomicCmpxchgOrRmwInfo(MI)) 1029 Changed |= expandAtomicCmpxchgOrRmw(MOI.getValue(), MI); 1030 } 1031 } 1032 1033 Changed |= removeAtomicPseudoMIs(); 1034 return Changed; 1035 } 1036 1037 INITIALIZE_PASS(SIMemoryLegalizer, DEBUG_TYPE, PASS_NAME, false, false) 1038 1039 char SIMemoryLegalizer::ID = 0; 1040 char &llvm::SIMemoryLegalizerID = SIMemoryLegalizer::ID; 1041 1042 FunctionPass *llvm::createSIMemoryLegalizerPass() { 1043 return new SIMemoryLegalizer(); 1044 } 1045