Home | History | Annotate | Download | only in AMDGPU 
      1 //===-- SIOptimizeExecMasking.cpp -----------------------------------------===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 
     10 #include "AMDGPU.h"
     11 #include "AMDGPUSubtarget.h"
     12 #include "SIInstrInfo.h"
     13 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
     14 #include "llvm/ADT/SmallSet.h"
     15 #include "llvm/CodeGen/MachineFunctionPass.h"
     16 #include "llvm/CodeGen/MachineInstrBuilder.h"
     17 #include "llvm/CodeGen/MachineRegisterInfo.h"
     18 #include "llvm/Support/Debug.h"
     19 
     20 using namespace llvm;
     21 
     22 #define DEBUG_TYPE "si-optimize-exec-masking"
     23 
     24 namespace {
     25 
     26 class SIOptimizeExecMasking : public MachineFunctionPass {
     27 public:
     28   static char ID;
     29 
     30 public:
     31   SIOptimizeExecMasking() : MachineFunctionPass(ID) {
     32     initializeSIOptimizeExecMaskingPass(*PassRegistry::getPassRegistry());
     33   }
     34 
     35   bool runOnMachineFunction(MachineFunction &MF) override;
     36 
     37   StringRef getPassName() const override {
     38     return "SI optimize exec mask operations";
     39   }
     40 
     41   void getAnalysisUsage(AnalysisUsage &AU) const override {
     42     AU.setPreservesCFG();
     43     MachineFunctionPass::getAnalysisUsage(AU);
     44   }
     45 };
     46 
     47 } // End anonymous namespace.
     48 
     49 INITIALIZE_PASS_BEGIN(SIOptimizeExecMasking, DEBUG_TYPE,
     50                       "SI optimize exec mask operations", false, false)
     51 INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
     52 INITIALIZE_PASS_END(SIOptimizeExecMasking, DEBUG_TYPE,
     53                     "SI optimize exec mask operations", false, false)
     54 
     55 char SIOptimizeExecMasking::ID = 0;
     56 
     57 char &llvm::SIOptimizeExecMaskingID = SIOptimizeExecMasking::ID;
     58 
     59 /// If \p MI is a copy from exec, return the register copied to.
     60 static unsigned isCopyFromExec(const MachineInstr &MI) {
     61   switch (MI.getOpcode()) {
     62   case AMDGPU::COPY:
     63   case AMDGPU::S_MOV_B64:
     64   case AMDGPU::S_MOV_B64_term: {
     65     const MachineOperand &Src = MI.getOperand(1);
     66     if (Src.isReg() && Src.getReg() == AMDGPU::EXEC)
     67       return MI.getOperand(0).getReg();
     68   }
     69   }
     70 
     71   return AMDGPU::NoRegister;
     72 }
     73 
     74 /// If \p MI is a copy to exec, return the register copied from.
     75 static unsigned isCopyToExec(const MachineInstr &MI) {
     76   switch (MI.getOpcode()) {
     77   case AMDGPU::COPY:
     78   case AMDGPU::S_MOV_B64: {
     79     const MachineOperand &Dst = MI.getOperand(0);
     80     if (Dst.isReg() && Dst.getReg() == AMDGPU::EXEC && MI.getOperand(1).isReg())
     81       return MI.getOperand(1).getReg();
     82     break;
     83   }
     84   case AMDGPU::S_MOV_B64_term:
     85     llvm_unreachable("should have been replaced");
     86   }
     87 
     88   return AMDGPU::NoRegister;
     89 }
     90 
     91 /// If \p MI is a logical operation on an exec value,
     92 /// return the register copied to.
     93 static unsigned isLogicalOpOnExec(const MachineInstr &MI) {
     94   switch (MI.getOpcode()) {
     95   case AMDGPU::S_AND_B64:
     96   case AMDGPU::S_OR_B64:
     97   case AMDGPU::S_XOR_B64:
     98   case AMDGPU::S_ANDN2_B64:
     99   case AMDGPU::S_ORN2_B64:
    100   case AMDGPU::S_NAND_B64:
    101   case AMDGPU::S_NOR_B64:
    102   case AMDGPU::S_XNOR_B64: {
    103     const MachineOperand &Src1 = MI.getOperand(1);
    104     if (Src1.isReg() && Src1.getReg() == AMDGPU::EXEC)
    105       return MI.getOperand(0).getReg();
    106     const MachineOperand &Src2 = MI.getOperand(2);
    107     if (Src2.isReg() && Src2.getReg() == AMDGPU::EXEC)
    108       return MI.getOperand(0).getReg();
    109   }
    110   }
    111 
    112   return AMDGPU::NoRegister;
    113 }
    114 
    115 static unsigned getSaveExecOp(unsigned Opc) {
    116   switch (Opc) {
    117   case AMDGPU::S_AND_B64:
    118     return AMDGPU::S_AND_SAVEEXEC_B64;
    119   case AMDGPU::S_OR_B64:
    120     return AMDGPU::S_OR_SAVEEXEC_B64;
    121   case AMDGPU::S_XOR_B64:
    122     return AMDGPU::S_XOR_SAVEEXEC_B64;
    123   case AMDGPU::S_ANDN2_B64:
    124     return AMDGPU::S_ANDN2_SAVEEXEC_B64;
    125   case AMDGPU::S_ORN2_B64:
    126     return AMDGPU::S_ORN2_SAVEEXEC_B64;
    127   case AMDGPU::S_NAND_B64:
    128     return AMDGPU::S_NAND_SAVEEXEC_B64;
    129   case AMDGPU::S_NOR_B64:
    130     return AMDGPU::S_NOR_SAVEEXEC_B64;
    131   case AMDGPU::S_XNOR_B64:
    132     return AMDGPU::S_XNOR_SAVEEXEC_B64;
    133   default:
    134     return AMDGPU::INSTRUCTION_LIST_END;
    135   }
    136 }
    137 
    138 // These are only terminators to get correct spill code placement during
    139 // register allocation, so turn them back into normal instructions. Only one of
    140 // these is expected per block.
    141 static bool removeTerminatorBit(const SIInstrInfo &TII, MachineInstr &MI) {
    142   switch (MI.getOpcode()) {
    143   case AMDGPU::S_MOV_B64_term: {
    144     MI.setDesc(TII.get(AMDGPU::COPY));
    145     return true;
    146   }
    147   case AMDGPU::S_XOR_B64_term: {
    148     // This is only a terminator to get the correct spill code placement during
    149     // register allocation.
    150     MI.setDesc(TII.get(AMDGPU::S_XOR_B64));
    151     return true;
    152   }
    153   case AMDGPU::S_ANDN2_B64_term: {
    154     // This is only a terminator to get the correct spill code placement during
    155     // register allocation.
    156     MI.setDesc(TII.get(AMDGPU::S_ANDN2_B64));
    157     return true;
    158   }
    159   default:
    160     return false;
    161   }
    162 }
    163 
    164 static MachineBasicBlock::reverse_iterator fixTerminators(
    165   const SIInstrInfo &TII,
    166   MachineBasicBlock &MBB) {
    167   MachineBasicBlock::reverse_iterator I = MBB.rbegin(), E = MBB.rend();
    168   for (; I != E; ++I) {
    169     if (!I->isTerminator())
    170       return I;
    171 
    172     if (removeTerminatorBit(TII, *I))
    173       return I;
    174   }
    175 
    176   return E;
    177 }
    178 
    179 static MachineBasicBlock::reverse_iterator findExecCopy(
    180   const SIInstrInfo &TII,
    181   MachineBasicBlock &MBB,
    182   MachineBasicBlock::reverse_iterator I,
    183   unsigned CopyToExec) {
    184   const unsigned InstLimit = 25;
    185 
    186   auto E = MBB.rend();
    187   for (unsigned N = 0; N <= InstLimit && I != E; ++I, ++N) {
    188     unsigned CopyFromExec = isCopyFromExec(*I);
    189     if (CopyFromExec != AMDGPU::NoRegister)
    190       return I;
    191   }
    192 
    193   return E;
    194 }
    195 
    196 // XXX - Seems LivePhysRegs doesn't work correctly since it will incorrectly
    197 // repor tthe register as unavailable because a super-register with a lane mask
    198 // as unavailable.
    199 static bool isLiveOut(const MachineBasicBlock &MBB, unsigned Reg) {
    200   for (MachineBasicBlock *Succ : MBB.successors()) {
    201     if (Succ->isLiveIn(Reg))
    202       return true;
    203   }
    204 
    205   return false;
    206 }
    207 
    208 bool SIOptimizeExecMasking::runOnMachineFunction(MachineFunction &MF) {
    209   if (skipFunction(MF.getFunction()))
    210     return false;
    211 
    212   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
    213   const SIRegisterInfo *TRI = ST.getRegisterInfo();
    214   const SIInstrInfo *TII = ST.getInstrInfo();
    215 
    216   // Optimize sequences emitted for control flow lowering. They are originally
    217   // emitted as the separate operations because spill code may need to be
    218   // inserted for the saved copy of exec.
    219   //
    220   //     x = copy exec
    221   //     z = s_<op>_b64 x, y
    222   //     exec = copy z
    223   // =>
    224   //     x = s_<op>_saveexec_b64 y
    225   //
    226 
    227   for (MachineBasicBlock &MBB : MF) {
    228     MachineBasicBlock::reverse_iterator I = fixTerminators(*TII, MBB);
    229     MachineBasicBlock::reverse_iterator E = MBB.rend();
    230     if (I == E)
    231       continue;
    232 
    233     unsigned CopyToExec = isCopyToExec(*I);
    234     if (CopyToExec == AMDGPU::NoRegister)
    235       continue;
    236 
    237     // Scan backwards to find the def.
    238     auto CopyToExecInst = &*I;
    239     auto CopyFromExecInst = findExecCopy(*TII, MBB, I, CopyToExec);
    240     if (CopyFromExecInst == E) {
    241       auto PrepareExecInst = std::next(I);
    242       if (PrepareExecInst == E)
    243         continue;
    244       // Fold exec = COPY (S_AND_B64 reg, exec) -> exec = S_AND_B64 reg, exec
    245       if (CopyToExecInst->getOperand(1).isKill() &&
    246           isLogicalOpOnExec(*PrepareExecInst) == CopyToExec) {
    247         LLVM_DEBUG(dbgs() << "Fold exec copy: " << *PrepareExecInst);
    248 
    249         PrepareExecInst->getOperand(0).setReg(AMDGPU::EXEC);
    250 
    251         LLVM_DEBUG(dbgs() << "into: " << *PrepareExecInst << '\n');
    252 
    253         CopyToExecInst->eraseFromParent();
    254       }
    255 
    256       continue;
    257     }
    258 
    259     if (isLiveOut(MBB, CopyToExec)) {
    260       // The copied register is live out and has a second use in another block.
    261       LLVM_DEBUG(dbgs() << "Exec copy source register is live out\n");
    262       continue;
    263     }
    264 
    265     unsigned CopyFromExec = CopyFromExecInst->getOperand(0).getReg();
    266     MachineInstr *SaveExecInst = nullptr;
    267     SmallVector<MachineInstr *, 4> OtherUseInsts;
    268 
    269     for (MachineBasicBlock::iterator J
    270            = std::next(CopyFromExecInst->getIterator()), JE = I->getIterator();
    271          J != JE; ++J) {
    272       if (SaveExecInst && J->readsRegister(AMDGPU::EXEC, TRI)) {
    273         LLVM_DEBUG(dbgs() << "exec read prevents saveexec: " << *J << '\n');
    274         // Make sure this is inserted after any VALU ops that may have been
    275         // scheduled in between.
    276         SaveExecInst = nullptr;
    277         break;
    278       }
    279 
    280       bool ReadsCopyFromExec = J->readsRegister(CopyFromExec, TRI);
    281 
    282       if (J->modifiesRegister(CopyToExec, TRI)) {
    283         if (SaveExecInst) {
    284           LLVM_DEBUG(dbgs() << "Multiple instructions modify "
    285                             << printReg(CopyToExec, TRI) << '\n');
    286           SaveExecInst = nullptr;
    287           break;
    288         }
    289 
    290         unsigned SaveExecOp = getSaveExecOp(J->getOpcode());
    291         if (SaveExecOp == AMDGPU::INSTRUCTION_LIST_END)
    292           break;
    293 
    294         if (ReadsCopyFromExec) {
    295           SaveExecInst = &*J;
    296           LLVM_DEBUG(dbgs() << "Found save exec op: " << *SaveExecInst << '\n');
    297           continue;
    298         } else {
    299           LLVM_DEBUG(dbgs()
    300                      << "Instruction does not read exec copy: " << *J << '\n');
    301           break;
    302         }
    303       } else if (ReadsCopyFromExec && !SaveExecInst) {
    304         // Make sure no other instruction is trying to use this copy, before it
    305         // will be rewritten by the saveexec, i.e. hasOneUse. There may have
    306         // been another use, such as an inserted spill. For example:
    307         //
    308         // %sgpr0_sgpr1 = COPY %exec
    309         // spill %sgpr0_sgpr1
    310         // %sgpr2_sgpr3 = S_AND_B64 %sgpr0_sgpr1
    311         //
    312         LLVM_DEBUG(dbgs() << "Found second use of save inst candidate: " << *J
    313                           << '\n');
    314         break;
    315       }
    316 
    317       if (SaveExecInst && J->readsRegister(CopyToExec, TRI)) {
    318         assert(SaveExecInst != &*J);
    319         OtherUseInsts.push_back(&*J);
    320       }
    321     }
    322 
    323     if (!SaveExecInst)
    324       continue;
    325 
    326     LLVM_DEBUG(dbgs() << "Insert save exec op: " << *SaveExecInst << '\n');
    327 
    328     MachineOperand &Src0 = SaveExecInst->getOperand(1);
    329     MachineOperand &Src1 = SaveExecInst->getOperand(2);
    330 
    331     MachineOperand *OtherOp = nullptr;
    332 
    333     if (Src0.isReg() && Src0.getReg() == CopyFromExec) {
    334       OtherOp = &Src1;
    335     } else if (Src1.isReg() && Src1.getReg() == CopyFromExec) {
    336       if (!SaveExecInst->isCommutable())
    337         break;
    338 
    339       OtherOp = &Src0;
    340     } else
    341       llvm_unreachable("unexpected");
    342 
    343     CopyFromExecInst->eraseFromParent();
    344 
    345     auto InsPt = SaveExecInst->getIterator();
    346     const DebugLoc &DL = SaveExecInst->getDebugLoc();
    347 
    348     BuildMI(MBB, InsPt, DL, TII->get(getSaveExecOp(SaveExecInst->getOpcode())),
    349             CopyFromExec)
    350       .addReg(OtherOp->getReg());
    351     SaveExecInst->eraseFromParent();
    352 
    353     CopyToExecInst->eraseFromParent();
    354 
    355     for (MachineInstr *OtherInst : OtherUseInsts) {
    356       OtherInst->substituteRegister(CopyToExec, AMDGPU::EXEC,
    357                                     AMDGPU::NoSubRegister, *TRI);
    358     }
    359   }
    360 
    361   return true;
    362 
    363 }
    364