Home | History | Annotate | Download | only in AMDGPU 
      1 //===-- SIFormMemoryClauses.cpp -------------------------------------------===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 /// \file
     11 /// This pass creates bundles of SMEM and VMEM instructions forming memory
     12 /// clauses if XNACK is enabled. Def operands of clauses are marked as early
     13 /// clobber to make sure we will not override any source within a clause.
     14 ///
     15 //===----------------------------------------------------------------------===//
     16 
     17 #include "AMDGPU.h"
     18 #include "AMDGPUSubtarget.h"
     19 #include "GCNRegPressure.h"
     20 #include "SIInstrInfo.h"
     21 #include "SIMachineFunctionInfo.h"
     22 #include "SIRegisterInfo.h"
     23 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
     24 #include "llvm/ADT/DenseMap.h"
     25 #include "llvm/CodeGen/LiveIntervals.h"
     26 #include "llvm/CodeGen/MachineFunctionPass.h"
     27 
     28 using namespace llvm;
     29 
     30 #define DEBUG_TYPE "si-form-memory-clauses"
     31 
     32 // Clauses longer then 15 instructions would overflow one of the counters
     33 // and stall. They can stall even earlier if there are outstanding counters.
     34 static cl::opt<unsigned>
     35 MaxClause("amdgpu-max-memory-clause", cl::Hidden, cl::init(15),
     36           cl::desc("Maximum length of a memory clause, instructions"));
     37 
     38 namespace {
     39 
     40 class SIFormMemoryClauses : public MachineFunctionPass {
     41   typedef DenseMap<unsigned, std::pair<unsigned, LaneBitmask>> RegUse;
     42 
     43 public:
     44   static char ID;
     45 
     46 public:
     47   SIFormMemoryClauses() : MachineFunctionPass(ID) {
     48     initializeSIFormMemoryClausesPass(*PassRegistry::getPassRegistry());
     49   }
     50 
     51   bool runOnMachineFunction(MachineFunction &MF) override;
     52 
     53   StringRef getPassName() const override {
     54     return "SI Form memory clauses";
     55   }
     56 
     57   void getAnalysisUsage(AnalysisUsage &AU) const override {
     58     AU.addRequired<LiveIntervals>();
     59     AU.setPreservesAll();
     60     MachineFunctionPass::getAnalysisUsage(AU);
     61   }
     62 
     63 private:
     64   template <typename Callable>
     65   void forAllLanes(unsigned Reg, LaneBitmask LaneMask, Callable Func) const;
     66 
     67   bool canBundle(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
     68   bool checkPressure(const MachineInstr &MI, GCNDownwardRPTracker &RPT);
     69   void collectRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
     70   bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses,
     71                       GCNDownwardRPTracker &RPT);
     72 
     73   const GCNSubtarget *ST;
     74   const SIRegisterInfo *TRI;
     75   const MachineRegisterInfo *MRI;
     76   SIMachineFunctionInfo *MFI;
     77 
     78   unsigned LastRecordedOccupancy;
     79   unsigned MaxVGPRs;
     80   unsigned MaxSGPRs;
     81 };
     82 
     83 } // End anonymous namespace.
     84 
     85 INITIALIZE_PASS_BEGIN(SIFormMemoryClauses, DEBUG_TYPE,
     86                       "SI Form memory clauses", false, false)
     87 INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
     88 INITIALIZE_PASS_END(SIFormMemoryClauses, DEBUG_TYPE,
     89                     "SI Form memory clauses", false, false)
     90 
     91 
     92 char SIFormMemoryClauses::ID = 0;
     93 
     94 char &llvm::SIFormMemoryClausesID = SIFormMemoryClauses::ID;
     95 
     96 FunctionPass *llvm::createSIFormMemoryClausesPass() {
     97   return new SIFormMemoryClauses();
     98 }
     99 
    100 static bool isVMEMClauseInst(const MachineInstr &MI) {
    101   return SIInstrInfo::isFLAT(MI) || SIInstrInfo::isVMEM(MI);
    102 }
    103 
    104 static bool isSMEMClauseInst(const MachineInstr &MI) {
    105   return SIInstrInfo::isSMRD(MI);
    106 }
    107 
    108 // There no sense to create store clauses, they do not define anything,
    109 // thus there is nothing to set early-clobber.
    110 static bool isValidClauseInst(const MachineInstr &MI, bool IsVMEMClause) {
    111   if (MI.isDebugValue() || MI.isBundled())
    112     return false;
    113   if (!MI.mayLoad() || MI.mayStore())
    114     return false;
    115   if (AMDGPU::getAtomicNoRetOp(MI.getOpcode()) != -1 ||
    116       AMDGPU::getAtomicRetOp(MI.getOpcode()) != -1)
    117     return false;
    118   if (IsVMEMClause && !isVMEMClauseInst(MI))
    119     return false;
    120   if (!IsVMEMClause && !isSMEMClauseInst(MI))
    121     return false;
    122   return true;
    123 }
    124 
    125 static unsigned getMopState(const MachineOperand &MO) {
    126   unsigned S = 0;
    127   if (MO.isImplicit())
    128     S |= RegState::Implicit;
    129   if (MO.isDead())
    130     S |= RegState::Dead;
    131   if (MO.isUndef())
    132     S |= RegState::Undef;
    133   if (MO.isKill())
    134     S |= RegState::Kill;
    135   if (MO.isEarlyClobber())
    136     S |= RegState::EarlyClobber;
    137   if (TargetRegisterInfo::isPhysicalRegister(MO.getReg()) && MO.isRenamable())
    138     S |= RegState::Renamable;
    139   return S;
    140 }
    141 
    142 template <typename Callable>
    143 void SIFormMemoryClauses::forAllLanes(unsigned Reg, LaneBitmask LaneMask,
    144                                       Callable Func) const {
    145   if (LaneMask.all() || TargetRegisterInfo::isPhysicalRegister(Reg) ||
    146       LaneMask == MRI->getMaxLaneMaskForVReg(Reg)) {
    147     Func(0);
    148     return;
    149   }
    150 
    151   const TargetRegisterClass *RC = MRI->getRegClass(Reg);
    152   unsigned E = TRI->getNumSubRegIndices();
    153   SmallVector<unsigned, AMDGPU::NUM_TARGET_SUBREGS> CoveringSubregs;
    154   for (unsigned Idx = 1; Idx < E; ++Idx) {
    155     // Is this index even compatible with the given class?
    156     if (TRI->getSubClassWithSubReg(RC, Idx) != RC)
    157       continue;
    158     LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx);
    159     // Early exit if we found a perfect match.
    160     if (SubRegMask == LaneMask) {
    161       Func(Idx);
    162       return;
    163     }
    164 
    165     if ((SubRegMask & ~LaneMask).any() || (SubRegMask & LaneMask).none())
    166       continue;
    167 
    168     CoveringSubregs.push_back(Idx);
    169   }
    170 
    171   llvm::sort(CoveringSubregs.begin(), CoveringSubregs.end(),
    172              [this](unsigned A, unsigned B) {
    173                LaneBitmask MaskA = TRI->getSubRegIndexLaneMask(A);
    174                LaneBitmask MaskB = TRI->getSubRegIndexLaneMask(B);
    175                unsigned NA = MaskA.getNumLanes();
    176                unsigned NB = MaskB.getNumLanes();
    177                if (NA != NB)
    178                  return NA > NB;
    179                return MaskA.getHighestLane() > MaskB.getHighestLane();
    180              });
    181 
    182   for (unsigned Idx : CoveringSubregs) {
    183     LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx);
    184     if ((SubRegMask & ~LaneMask).any() || (SubRegMask & LaneMask).none())
    185       continue;
    186 
    187     Func(Idx);
    188     LaneMask &= ~SubRegMask;
    189     if (LaneMask.none())
    190       return;
    191   }
    192 
    193   llvm_unreachable("Failed to find all subregs to cover lane mask");
    194 }
    195 
    196 // Returns false if there is a use of a def already in the map.
    197 // In this case we must break the clause.
    198 bool SIFormMemoryClauses::canBundle(const MachineInstr &MI,
    199                                     RegUse &Defs, RegUse &Uses) const {
    200   // Check interference with defs.
    201   for (const MachineOperand &MO : MI.operands()) {
    202     // TODO: Prologue/Epilogue Insertion pass does not process bundled
    203     //       instructions.
    204     if (MO.isFI())
    205       return false;
    206 
    207     if (!MO.isReg())
    208       continue;
    209 
    210     unsigned Reg = MO.getReg();
    211 
    212     // If it is tied we will need to write same register as we read.
    213     if (MO.isTied())
    214       return false;
    215 
    216     RegUse &Map = MO.isDef() ? Uses : Defs;
    217     auto Conflict = Map.find(Reg);
    218     if (Conflict == Map.end())
    219       continue;
    220 
    221     if (TargetRegisterInfo::isPhysicalRegister(Reg))
    222       return false;
    223 
    224     LaneBitmask Mask = TRI->getSubRegIndexLaneMask(MO.getSubReg());
    225     if ((Conflict->second.second & Mask).any())
    226       return false;
    227   }
    228 
    229   return true;
    230 }
    231 
    232 // Since all defs in the clause are early clobber we can run out of registers.
    233 // Function returns false if pressure would hit the limit if instruction is
    234 // bundled into a memory clause.
    235 bool SIFormMemoryClauses::checkPressure(const MachineInstr &MI,
    236                                         GCNDownwardRPTracker &RPT) {
    237   // NB: skip advanceBeforeNext() call. Since all defs will be marked
    238   // early-clobber they will all stay alive at least to the end of the
    239   // clause. Therefor we should not decrease pressure even if load
    240   // pointer becomes dead and could otherwise be reused for destination.
    241   RPT.advanceToNext();
    242   GCNRegPressure MaxPressure = RPT.moveMaxPressure();
    243   unsigned Occupancy = MaxPressure.getOccupancy(*ST);
    244   if (Occupancy >= MFI->getMinAllowedOccupancy() &&
    245       MaxPressure.getVGPRNum() <= MaxVGPRs &&
    246       MaxPressure.getSGPRNum() <= MaxSGPRs) {
    247     LastRecordedOccupancy = Occupancy;
    248     return true;
    249   }
    250   return false;
    251 }
    252 
    253 // Collect register defs and uses along with their lane masks and states.
    254 void SIFormMemoryClauses::collectRegUses(const MachineInstr &MI,
    255                                          RegUse &Defs, RegUse &Uses) const {
    256   for (const MachineOperand &MO : MI.operands()) {
    257     if (!MO.isReg())
    258       continue;
    259     unsigned Reg = MO.getReg();
    260     if (!Reg)
    261       continue;
    262 
    263     LaneBitmask Mask = TargetRegisterInfo::isVirtualRegister(Reg) ?
    264                          TRI->getSubRegIndexLaneMask(MO.getSubReg()) :
    265                          LaneBitmask::getAll();
    266     RegUse &Map = MO.isDef() ? Defs : Uses;
    267 
    268     auto Loc = Map.find(Reg);
    269     unsigned State = getMopState(MO);
    270     if (Loc == Map.end()) {
    271       Map[Reg] = std::make_pair(State, Mask);
    272     } else {
    273       Loc->second.first |= State;
    274       Loc->second.second |= Mask;
    275     }
    276   }
    277 }
    278 
    279 // Check register def/use conflicts, occupancy limits and collect def/use maps.
    280 // Return true if instruction can be bundled with previous. It it cannot
    281 // def/use maps are not updated.
    282 bool SIFormMemoryClauses::processRegUses(const MachineInstr &MI,
    283                                          RegUse &Defs, RegUse &Uses,
    284                                          GCNDownwardRPTracker &RPT) {
    285   if (!canBundle(MI, Defs, Uses))
    286     return false;
    287 
    288   if (!checkPressure(MI, RPT))
    289     return false;
    290 
    291   collectRegUses(MI, Defs, Uses);
    292   return true;
    293 }
    294 
    295 bool SIFormMemoryClauses::runOnMachineFunction(MachineFunction &MF) {
    296   if (skipFunction(MF.getFunction()))
    297     return false;
    298 
    299   ST = &MF.getSubtarget<GCNSubtarget>();
    300   if (!ST->isXNACKEnabled())
    301     return false;
    302 
    303   const SIInstrInfo *TII = ST->getInstrInfo();
    304   TRI = ST->getRegisterInfo();
    305   MRI = &MF.getRegInfo();
    306   MFI = MF.getInfo<SIMachineFunctionInfo>();
    307   LiveIntervals *LIS = &getAnalysis<LiveIntervals>();
    308   SlotIndexes *Ind = LIS->getSlotIndexes();
    309   bool Changed = false;
    310 
    311   MaxVGPRs = TRI->getAllocatableSet(MF, &AMDGPU::VGPR_32RegClass).count();
    312   MaxSGPRs = TRI->getAllocatableSet(MF, &AMDGPU::SGPR_32RegClass).count();
    313 
    314   for (MachineBasicBlock &MBB : MF) {
    315     MachineBasicBlock::instr_iterator Next;
    316     for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; I = Next) {
    317       MachineInstr &MI = *I;
    318       Next = std::next(I);
    319 
    320       bool IsVMEM = isVMEMClauseInst(MI);
    321 
    322       if (!isValidClauseInst(MI, IsVMEM))
    323         continue;
    324 
    325       RegUse Defs, Uses;
    326       GCNDownwardRPTracker RPT(*LIS);
    327       RPT.reset(MI);
    328 
    329       if (!processRegUses(MI, Defs, Uses, RPT))
    330         continue;
    331 
    332       unsigned Length = 1;
    333       for ( ; Next != E && Length < MaxClause; ++Next) {
    334         if (!isValidClauseInst(*Next, IsVMEM))
    335           break;
    336 
    337         // A load from pointer which was loaded inside the same bundle is an
    338         // impossible clause because we will need to write and read the same
    339         // register inside. In this case processRegUses will return false.
    340         if (!processRegUses(*Next, Defs, Uses, RPT))
    341           break;
    342 
    343         ++Length;
    344       }
    345       if (Length < 2)
    346         continue;
    347 
    348       Changed = true;
    349       MFI->limitOccupancy(LastRecordedOccupancy);
    350 
    351       auto B = BuildMI(MBB, I, DebugLoc(), TII->get(TargetOpcode::BUNDLE));
    352       Ind->insertMachineInstrInMaps(*B);
    353 
    354       for (auto BI = I; BI != Next; ++BI) {
    355         BI->bundleWithPred();
    356         Ind->removeSingleMachineInstrFromMaps(*BI);
    357 
    358         for (MachineOperand &MO : BI->defs())
    359           if (MO.readsReg())
    360             MO.setIsInternalRead(true);
    361       }
    362 
    363       for (auto &&R : Defs) {
    364         forAllLanes(R.first, R.second.second, [&R, &B](unsigned SubReg) {
    365           unsigned S = R.second.first | RegState::EarlyClobber;
    366           if (!SubReg)
    367             S &= ~(RegState::Undef | RegState::Dead);
    368           B.addDef(R.first, S, SubReg);
    369         });
    370       }
    371 
    372       for (auto &&R : Uses) {
    373         forAllLanes(R.first, R.second.second, [&R, &B](unsigned SubReg) {
    374           B.addUse(R.first, R.second.first & ~RegState::Kill, SubReg);
    375         });
    376       }
    377 
    378       for (auto &&R : Defs) {
    379         unsigned Reg = R.first;
    380         Uses.erase(Reg);
    381         if (TargetRegisterInfo::isPhysicalRegister(Reg))
    382           continue;
    383         LIS->removeInterval(Reg);
    384         LIS->createAndComputeVirtRegInterval(Reg);
    385       }
    386 
    387       for (auto &&R : Uses) {
    388         unsigned Reg = R.first;
    389         if (TargetRegisterInfo::isPhysicalRegister(Reg))
    390           continue;
    391         LIS->removeInterval(Reg);
    392         LIS->createAndComputeVirtRegInterval(Reg);
    393       }
    394     }
    395   }
    396 
    397   return Changed;
    398 }
    399