Home | History | Annotate | Download | only in ARM 
      1 //===-- MLxExpansionPass.cpp - Expand MLx instrs to avoid hazards ---------===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 // Expand VFP / NEON floating point MLA / MLS instructions (each to a pair of
     11 // multiple and add / sub instructions) when special VMLx hazards are detected.
     12 //
     13 //===----------------------------------------------------------------------===//
     14 
     15 #include "ARM.h"
     16 #include "ARMBaseInstrInfo.h"
     17 #include "ARMSubtarget.h"
     18 #include "llvm/ADT/SmallPtrSet.h"
     19 #include "llvm/ADT/Statistic.h"
     20 #include "llvm/CodeGen/MachineFunctionPass.h"
     21 #include "llvm/CodeGen/MachineInstr.h"
     22 #include "llvm/CodeGen/MachineInstrBuilder.h"
     23 #include "llvm/CodeGen/MachineRegisterInfo.h"
     24 #include "llvm/Support/CommandLine.h"
     25 #include "llvm/Support/Debug.h"
     26 #include "llvm/Support/raw_ostream.h"
     27 #include "llvm/Target/TargetRegisterInfo.h"
     28 using namespace llvm;
     29 
     30 #define DEBUG_TYPE "mlx-expansion"
     31 
     32 static cl::opt<bool>
     33 ForceExapnd("expand-all-fp-mlx", cl::init(false), cl::Hidden);
     34 static cl::opt<unsigned>
     35 ExpandLimit("expand-limit", cl::init(~0U), cl::Hidden);
     36 
     37 STATISTIC(NumExpand, "Number of fp MLA / MLS instructions expanded");
     38 
     39 namespace {
     40   struct MLxExpansion : public MachineFunctionPass {
     41     static char ID;
     42     MLxExpansion() : MachineFunctionPass(ID) {}
     43 
     44     bool runOnMachineFunction(MachineFunction &Fn) override;
     45 
     46     const char *getPassName() const override {
     47       return "ARM MLA / MLS expansion pass";
     48     }
     49 
     50   private:
     51     const ARMBaseInstrInfo *TII;
     52     const TargetRegisterInfo *TRI;
     53     MachineRegisterInfo *MRI;
     54 
     55     bool isLikeA9;
     56     bool isSwift;
     57     unsigned MIIdx;
     58     MachineInstr* LastMIs[4];
     59     SmallPtrSet<MachineInstr*, 4> IgnoreStall;
     60 
     61     void clearStack();
     62     void pushStack(MachineInstr *MI);
     63     MachineInstr *getAccDefMI(MachineInstr *MI) const;
     64     unsigned getDefReg(MachineInstr *MI) const;
     65     bool hasLoopHazard(MachineInstr *MI) const;
     66     bool hasRAWHazard(unsigned Reg, MachineInstr *MI) const;
     67     bool FindMLxHazard(MachineInstr *MI);
     68     void ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
     69                                 unsigned MulOpc, unsigned AddSubOpc,
     70                                 bool NegAcc, bool HasLane);
     71     bool ExpandFPMLxInstructions(MachineBasicBlock &MBB);
     72   };
     73   char MLxExpansion::ID = 0;
     74 }
     75 
     76 void MLxExpansion::clearStack() {
     77   std::fill(LastMIs, LastMIs + 4, nullptr);
     78   MIIdx = 0;
     79 }
     80 
     81 void MLxExpansion::pushStack(MachineInstr *MI) {
     82   LastMIs[MIIdx] = MI;
     83   if (++MIIdx == 4)
     84     MIIdx = 0;
     85 }
     86 
     87 MachineInstr *MLxExpansion::getAccDefMI(MachineInstr *MI) const {
     88   // Look past COPY and INSERT_SUBREG instructions to find the
     89   // real definition MI. This is important for _sfp instructions.
     90   unsigned Reg = MI->getOperand(1).getReg();
     91   if (TargetRegisterInfo::isPhysicalRegister(Reg))
     92     return nullptr;
     93 
     94   MachineBasicBlock *MBB = MI->getParent();
     95   MachineInstr *DefMI = MRI->getVRegDef(Reg);
     96   while (true) {
     97     if (DefMI->getParent() != MBB)
     98       break;
     99     if (DefMI->isCopyLike()) {
    100       Reg = DefMI->getOperand(1).getReg();
    101       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
    102         DefMI = MRI->getVRegDef(Reg);
    103         continue;
    104       }
    105     } else if (DefMI->isInsertSubreg()) {
    106       Reg = DefMI->getOperand(2).getReg();
    107       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
    108         DefMI = MRI->getVRegDef(Reg);
    109         continue;
    110       }
    111     }
    112     break;
    113   }
    114   return DefMI;
    115 }
    116 
    117 unsigned MLxExpansion::getDefReg(MachineInstr *MI) const {
    118   unsigned Reg = MI->getOperand(0).getReg();
    119   if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
    120       !MRI->hasOneNonDBGUse(Reg))
    121     return Reg;
    122 
    123   MachineBasicBlock *MBB = MI->getParent();
    124   MachineInstr *UseMI = &*MRI->use_instr_nodbg_begin(Reg);
    125   if (UseMI->getParent() != MBB)
    126     return Reg;
    127 
    128   while (UseMI->isCopy() || UseMI->isInsertSubreg()) {
    129     Reg = UseMI->getOperand(0).getReg();
    130     if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
    131         !MRI->hasOneNonDBGUse(Reg))
    132       return Reg;
    133     UseMI = &*MRI->use_instr_nodbg_begin(Reg);
    134     if (UseMI->getParent() != MBB)
    135       return Reg;
    136   }
    137 
    138   return Reg;
    139 }
    140 
    141 /// hasLoopHazard - Check whether an MLx instruction is chained to itself across
    142 /// a single-MBB loop.
    143 bool MLxExpansion::hasLoopHazard(MachineInstr *MI) const {
    144   unsigned Reg = MI->getOperand(1).getReg();
    145   if (TargetRegisterInfo::isPhysicalRegister(Reg))
    146     return false;
    147 
    148   MachineBasicBlock *MBB = MI->getParent();
    149   MachineInstr *DefMI = MRI->getVRegDef(Reg);
    150   while (true) {
    151 outer_continue:
    152     if (DefMI->getParent() != MBB)
    153       break;
    154 
    155     if (DefMI->isPHI()) {
    156       for (unsigned i = 1, e = DefMI->getNumOperands(); i < e; i += 2) {
    157         if (DefMI->getOperand(i + 1).getMBB() == MBB) {
    158           unsigned SrcReg = DefMI->getOperand(i).getReg();
    159           if (TargetRegisterInfo::isVirtualRegister(SrcReg)) {
    160             DefMI = MRI->getVRegDef(SrcReg);
    161             goto outer_continue;
    162           }
    163         }
    164       }
    165     } else if (DefMI->isCopyLike()) {
    166       Reg = DefMI->getOperand(1).getReg();
    167       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
    168         DefMI = MRI->getVRegDef(Reg);
    169         continue;
    170       }
    171     } else if (DefMI->isInsertSubreg()) {
    172       Reg = DefMI->getOperand(2).getReg();
    173       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
    174         DefMI = MRI->getVRegDef(Reg);
    175         continue;
    176       }
    177     }
    178 
    179     break;
    180   }
    181 
    182   return DefMI == MI;
    183 }
    184 
    185 bool MLxExpansion::hasRAWHazard(unsigned Reg, MachineInstr *MI) const {
    186   // FIXME: Detect integer instructions properly.
    187   const MCInstrDesc &MCID = MI->getDesc();
    188   unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
    189   if (MI->mayStore())
    190     return false;
    191   unsigned Opcode = MCID.getOpcode();
    192   if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
    193     return false;
    194   if ((Domain & ARMII::DomainVFP) || (Domain & ARMII::DomainNEON))
    195     return MI->readsRegister(Reg, TRI);
    196   return false;
    197 }
    198 
    199 static bool isFpMulInstruction(unsigned Opcode) {
    200   switch (Opcode) {
    201   case ARM::VMULS:
    202   case ARM::VMULfd:
    203   case ARM::VMULfq:
    204   case ARM::VMULD:
    205   case ARM::VMULslfd:
    206   case ARM::VMULslfq:
    207     return true;
    208   default:
    209     return false;
    210   }
    211 }
    212 
    213 bool MLxExpansion::FindMLxHazard(MachineInstr *MI) {
    214   if (NumExpand >= ExpandLimit)
    215     return false;
    216 
    217   if (ForceExapnd)
    218     return true;
    219 
    220   MachineInstr *DefMI = getAccDefMI(MI);
    221   if (TII->isFpMLxInstruction(DefMI->getOpcode())) {
    222     // r0 = vmla
    223     // r3 = vmla r0, r1, r2
    224     // takes 16 - 17 cycles
    225     //
    226     // r0 = vmla
    227     // r4 = vmul r1, r2
    228     // r3 = vadd r0, r4
    229     // takes about 14 - 15 cycles even with vmul stalling for 4 cycles.
    230     IgnoreStall.insert(DefMI);
    231     return true;
    232   }
    233 
    234   // On Swift, we mostly care about hazards from multiplication instructions
    235   // writing the accumulator and the pipelining of loop iterations by out-of-
    236   // order execution.
    237   if (isSwift)
    238     return isFpMulInstruction(DefMI->getOpcode()) || hasLoopHazard(MI);
    239 
    240   if (IgnoreStall.count(MI))
    241     return false;
    242 
    243   // If a VMLA.F is followed by an VADD.F or VMUL.F with no RAW hazard, the
    244   // VADD.F or VMUL.F will stall 4 cycles before issue. The 4 cycle stall
    245   // preserves the in-order retirement of the instructions.
    246   // Look at the next few instructions, if *most* of them can cause hazards,
    247   // then the scheduler can't *fix* this, we'd better break up the VMLA.
    248   unsigned Limit1 = isLikeA9 ? 1 : 4;
    249   unsigned Limit2 = isLikeA9 ? 1 : 4;
    250   for (unsigned i = 1; i <= 4; ++i) {
    251     int Idx = ((int)MIIdx - i + 4) % 4;
    252     MachineInstr *NextMI = LastMIs[Idx];
    253     if (!NextMI)
    254       continue;
    255 
    256     if (TII->canCauseFpMLxStall(NextMI->getOpcode())) {
    257       if (i <= Limit1)
    258         return true;
    259     }
    260 
    261     // Look for VMLx RAW hazard.
    262     if (i <= Limit2 && hasRAWHazard(getDefReg(MI), NextMI))
    263       return true;
    264   }
    265 
    266   return false;
    267 }
    268 
    269 /// ExpandFPMLxInstructions - Expand a MLA / MLS instruction into a pair
    270 /// of MUL + ADD / SUB instructions.
    271 void
    272 MLxExpansion::ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
    273                                      unsigned MulOpc, unsigned AddSubOpc,
    274                                      bool NegAcc, bool HasLane) {
    275   unsigned DstReg = MI->getOperand(0).getReg();
    276   bool DstDead = MI->getOperand(0).isDead();
    277   unsigned AccReg = MI->getOperand(1).getReg();
    278   unsigned Src1Reg = MI->getOperand(2).getReg();
    279   unsigned Src2Reg = MI->getOperand(3).getReg();
    280   bool Src1Kill = MI->getOperand(2).isKill();
    281   bool Src2Kill = MI->getOperand(3).isKill();
    282   unsigned LaneImm = HasLane ? MI->getOperand(4).getImm() : 0;
    283   unsigned NextOp = HasLane ? 5 : 4;
    284   ARMCC::CondCodes Pred = (ARMCC::CondCodes)MI->getOperand(NextOp).getImm();
    285   unsigned PredReg = MI->getOperand(++NextOp).getReg();
    286 
    287   const MCInstrDesc &MCID1 = TII->get(MulOpc);
    288   const MCInstrDesc &MCID2 = TII->get(AddSubOpc);
    289   const MachineFunction &MF = *MI->getParent()->getParent();
    290   unsigned TmpReg = MRI->createVirtualRegister(
    291                       TII->getRegClass(MCID1, 0, TRI, MF));
    292 
    293   MachineInstrBuilder MIB = BuildMI(MBB, MI, MI->getDebugLoc(), MCID1, TmpReg)
    294     .addReg(Src1Reg, getKillRegState(Src1Kill))
    295     .addReg(Src2Reg, getKillRegState(Src2Kill));
    296   if (HasLane)
    297     MIB.addImm(LaneImm);
    298   MIB.addImm(Pred).addReg(PredReg);
    299 
    300   MIB = BuildMI(MBB, MI, MI->getDebugLoc(), MCID2)
    301     .addReg(DstReg, getDefRegState(true) | getDeadRegState(DstDead));
    302 
    303   if (NegAcc) {
    304     bool AccKill = MRI->hasOneNonDBGUse(AccReg);
    305     MIB.addReg(TmpReg, getKillRegState(true))
    306        .addReg(AccReg, getKillRegState(AccKill));
    307   } else {
    308     MIB.addReg(AccReg).addReg(TmpReg, getKillRegState(true));
    309   }
    310   MIB.addImm(Pred).addReg(PredReg);
    311 
    312   DEBUG({
    313       dbgs() << "Expanding: " << *MI;
    314       dbgs() << "  to:\n";
    315       MachineBasicBlock::iterator MII = MI;
    316       MII = std::prev(MII);
    317       MachineInstr &MI2 = *MII;
    318       MII = std::prev(MII);
    319       MachineInstr &MI1 = *MII;
    320       dbgs() << "    " << MI1;
    321       dbgs() << "    " << MI2;
    322    });
    323 
    324   MI->eraseFromParent();
    325   ++NumExpand;
    326 }
    327 
    328 bool MLxExpansion::ExpandFPMLxInstructions(MachineBasicBlock &MBB) {
    329   bool Changed = false;
    330 
    331   clearStack();
    332   IgnoreStall.clear();
    333 
    334   unsigned Skip = 0;
    335   MachineBasicBlock::reverse_iterator MII = MBB.rbegin(), E = MBB.rend();
    336   while (MII != E) {
    337     MachineInstr *MI = &*MII;
    338 
    339     if (MI->isPosition() || MI->isImplicitDef() || MI->isCopy()) {
    340       ++MII;
    341       continue;
    342     }
    343 
    344     const MCInstrDesc &MCID = MI->getDesc();
    345     if (MI->isBarrier()) {
    346       clearStack();
    347       Skip = 0;
    348       ++MII;
    349       continue;
    350     }
    351 
    352     unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
    353     if (Domain == ARMII::DomainGeneral) {
    354       if (++Skip == 2)
    355         // Assume dual issues of non-VFP / NEON instructions.
    356         pushStack(nullptr);
    357     } else {
    358       Skip = 0;
    359 
    360       unsigned MulOpc, AddSubOpc;
    361       bool NegAcc, HasLane;
    362       if (!TII->isFpMLxInstruction(MCID.getOpcode(),
    363                                    MulOpc, AddSubOpc, NegAcc, HasLane) ||
    364           !FindMLxHazard(MI))
    365         pushStack(MI);
    366       else {
    367         ExpandFPMLxInstruction(MBB, MI, MulOpc, AddSubOpc, NegAcc, HasLane);
    368         E = MBB.rend(); // May have changed if MI was the 1st instruction.
    369         Changed = true;
    370         continue;
    371       }
    372     }
    373 
    374     ++MII;
    375   }
    376 
    377   return Changed;
    378 }
    379 
    380 bool MLxExpansion::runOnMachineFunction(MachineFunction &Fn) {
    381   TII = static_cast<const ARMBaseInstrInfo *>(Fn.getSubtarget().getInstrInfo());
    382   TRI = Fn.getSubtarget().getRegisterInfo();
    383   MRI = &Fn.getRegInfo();
    384   const ARMSubtarget *STI = &Fn.getSubtarget<ARMSubtarget>();
    385   // Only run this for CortexA9.
    386   if (!STI->isCortexA9())
    387     return false;
    388   isLikeA9 = STI->isLikeA9() || STI->isSwift();
    389   isSwift = STI->isSwift();
    390 
    391   bool Modified = false;
    392   for (MachineBasicBlock &MBB : Fn)
    393     Modified |= ExpandFPMLxInstructions(MBB);
    394 
    395   return Modified;
    396 }
    397 
    398 FunctionPass *llvm::createMLxExpansionPass() {
    399   return new MLxExpansion();
    400 }
    401