Home | History | Annotate | Download | only in X86 
      1 //===-- X86VZeroUpper.cpp - AVX vzeroupper instruction inserter -----------===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 // This file defines the pass which inserts x86 AVX vzeroupper instructions
     11 // before calls to SSE encoded functions. This avoids transition latency
     12 // penalty when tranfering control between AVX encoded instructions and old
     13 // SSE encoding mode.
     14 //
     15 //===----------------------------------------------------------------------===//
     16 
     17 #define DEBUG_TYPE "x86-vzeroupper"
     18 #include "X86.h"
     19 #include "X86InstrInfo.h"
     20 #include "llvm/ADT/Statistic.h"
     21 #include "llvm/CodeGen/MachineFunctionPass.h"
     22 #include "llvm/CodeGen/MachineInstrBuilder.h"
     23 #include "llvm/CodeGen/MachineRegisterInfo.h"
     24 #include "llvm/CodeGen/Passes.h"
     25 #include "llvm/Support/Debug.h"
     26 #include "llvm/Support/raw_ostream.h"
     27 #include "llvm/Target/TargetInstrInfo.h"
     28 using namespace llvm;
     29 
     30 STATISTIC(NumVZU, "Number of vzeroupper instructions inserted");
     31 
     32 namespace {
     33   struct VZeroUpperInserter : public MachineFunctionPass {
     34     static char ID;
     35     VZeroUpperInserter() : MachineFunctionPass(ID) {}
     36 
     37     virtual bool runOnMachineFunction(MachineFunction &MF);
     38 
     39     bool processBasicBlock(MachineFunction &MF, MachineBasicBlock &MBB);
     40 
     41     virtual const char *getPassName() const { return "X86 vzeroupper inserter";}
     42 
     43   private:
     44     const TargetInstrInfo *TII; // Machine instruction info.
     45 
     46     // Any YMM register live-in to this function?
     47     bool FnHasLiveInYmm;
     48 
     49     // BBState - Contains the state of each MBB: unknown, clean, dirty
     50     SmallVector<uint8_t, 8> BBState;
     51 
     52     // BBSolved - Keep track of all MBB which had been already analyzed
     53     // and there is no further processing required.
     54     BitVector BBSolved;
     55 
     56     // Machine Basic Blocks are classified according this pass:
     57     //
     58     //  ST_UNKNOWN - The MBB state is unknown, meaning from the entry state
     59     //    until the MBB exit there isn't a instruction using YMM to change
     60     //    the state to dirty, or one of the incoming predecessors is unknown
     61     //    and there's not a dirty predecessor between them.
     62     //
     63     //  ST_CLEAN - No YMM usage in the end of the MBB. A MBB could have
     64     //    instructions using YMM and be marked ST_CLEAN, as long as the state
     65     //    is cleaned by a vzeroupper before any call.
     66     //
     67     //  ST_DIRTY - Any MBB ending with a YMM usage not cleaned up by a
     68     //    vzeroupper instruction.
     69     //
     70     //  ST_INIT - Placeholder for an empty state set
     71     //
     72     enum {
     73       ST_UNKNOWN = 0,
     74       ST_CLEAN   = 1,
     75       ST_DIRTY   = 2,
     76       ST_INIT    = 3
     77     };
     78 
     79     // computeState - Given two states, compute the resulting state, in
     80     // the following way
     81     //
     82     //  1) One dirty state yields another dirty state
     83     //  2) All states must be clean for the result to be clean
     84     //  3) If none above and one unknown, the result state is also unknown
     85     //
     86     static unsigned computeState(unsigned PrevState, unsigned CurState) {
     87       if (PrevState == ST_INIT)
     88         return CurState;
     89 
     90       if (PrevState == ST_DIRTY || CurState == ST_DIRTY)
     91         return ST_DIRTY;
     92 
     93       if (PrevState == ST_CLEAN && CurState == ST_CLEAN)
     94         return ST_CLEAN;
     95 
     96       return ST_UNKNOWN;
     97     }
     98 
     99   };
    100   char VZeroUpperInserter::ID = 0;
    101 }
    102 
    103 FunctionPass *llvm::createX86IssueVZeroUpperPass() {
    104   return new VZeroUpperInserter();
    105 }
    106 
    107 static bool isYmmReg(unsigned Reg) {
    108   if (Reg >= X86::YMM0 && Reg <= X86::YMM15)
    109     return true;
    110 
    111   return false;
    112 }
    113 
    114 static bool checkFnHasLiveInYmm(MachineRegisterInfo &MRI) {
    115   for (MachineRegisterInfo::livein_iterator I = MRI.livein_begin(),
    116        E = MRI.livein_end(); I != E; ++I)
    117     if (isYmmReg(I->first))
    118       return true;
    119 
    120   return false;
    121 }
    122 
    123 static bool clobbersAllYmmRegs(const MachineOperand &MO) {
    124   for (unsigned reg = X86::YMM0; reg < X86::YMM15; ++reg) {
    125     if (!MO.clobbersPhysReg(reg))
    126       return false;
    127   }
    128   return true;
    129 }
    130 
    131 static bool hasYmmReg(MachineInstr *MI) {
    132   for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
    133     const MachineOperand &MO = MI->getOperand(i);
    134     if (MI->isCall() && MO.isRegMask() && !clobbersAllYmmRegs(MO))
    135       return true;
    136     if (!MO.isReg())
    137       continue;
    138     if (MO.isDebug())
    139       continue;
    140     if (isYmmReg(MO.getReg()))
    141       return true;
    142   }
    143   return false;
    144 }
    145 
    146 /// runOnMachineFunction - Loop over all of the basic blocks, inserting
    147 /// vzero upper instructions before function calls.
    148 bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
    149   TII = MF.getTarget().getInstrInfo();
    150   MachineRegisterInfo &MRI = MF.getRegInfo();
    151   bool EverMadeChange = false;
    152 
    153   // Fast check: if the function doesn't use any ymm registers, we don't need
    154   // to insert any VZEROUPPER instructions.  This is constant-time, so it is
    155   // cheap in the common case of no ymm use.
    156   bool YMMUsed = false;
    157   const TargetRegisterClass *RC = &X86::VR256RegClass;
    158   for (TargetRegisterClass::iterator i = RC->begin(), e = RC->end();
    159        i != e; i++) {
    160     if (!MRI.reg_nodbg_empty(*i)) {
    161       YMMUsed = true;
    162       break;
    163     }
    164   }
    165   if (!YMMUsed)
    166     return EverMadeChange;
    167 
    168   // Pre-compute the existence of any live-in YMM registers to this function
    169   FnHasLiveInYmm = checkFnHasLiveInYmm(MRI);
    170 
    171   assert(BBState.empty());
    172   BBState.resize(MF.getNumBlockIDs(), 0);
    173   BBSolved.resize(MF.getNumBlockIDs(), 0);
    174 
    175   // Each BB state depends on all predecessors, loop over until everything
    176   // converges.  (Once we converge, we can implicitly mark everything that is
    177   // still ST_UNKNOWN as ST_CLEAN.)
    178   while (1) {
    179     bool MadeChange = false;
    180 
    181     // Process all basic blocks.
    182     for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I)
    183       MadeChange |= processBasicBlock(MF, *I);
    184 
    185     // If this iteration over the code changed anything, keep iterating.
    186     if (!MadeChange) break;
    187     EverMadeChange = true;
    188   }
    189 
    190   BBState.clear();
    191   BBSolved.clear();
    192   return EverMadeChange;
    193 }
    194 
    195 /// processBasicBlock - Loop over all of the instructions in the basic block,
    196 /// inserting vzero upper instructions before function calls.
    197 bool VZeroUpperInserter::processBasicBlock(MachineFunction &MF,
    198                                            MachineBasicBlock &BB) {
    199   bool Changed = false;
    200   unsigned BBNum = BB.getNumber();
    201 
    202   // Don't process already solved BBs
    203   if (BBSolved[BBNum])
    204     return false; // No changes
    205 
    206   // Check the state of all predecessors
    207   unsigned EntryState = ST_INIT;
    208   for (MachineBasicBlock::const_pred_iterator PI = BB.pred_begin(),
    209        PE = BB.pred_end(); PI != PE; ++PI) {
    210     EntryState = computeState(EntryState, BBState[(*PI)->getNumber()]);
    211     if (EntryState == ST_DIRTY)
    212       break;
    213   }
    214 
    215 
    216   // The entry MBB for the function may set the initial state to dirty if
    217   // the function receives any YMM incoming arguments
    218   if (&BB == MF.begin()) {
    219     EntryState = ST_CLEAN;
    220     if (FnHasLiveInYmm)
    221       EntryState = ST_DIRTY;
    222   }
    223 
    224   // The current state is initialized according to the predecessors
    225   unsigned CurState = EntryState;
    226   bool BBHasCall = false;
    227 
    228   for (MachineBasicBlock::iterator I = BB.begin(); I != BB.end(); ++I) {
    229     MachineInstr *MI = I;
    230     DebugLoc dl = I->getDebugLoc();
    231     bool isControlFlow = MI->isCall() || MI->isReturn();
    232 
    233     // Shortcut: don't need to check regular instructions in dirty state.
    234     if (!isControlFlow && CurState == ST_DIRTY)
    235       continue;
    236 
    237     if (hasYmmReg(MI)) {
    238       // We found a ymm-using instruction; this could be an AVX instruction,
    239       // or it could be control flow.
    240       CurState = ST_DIRTY;
    241       continue;
    242     }
    243 
    244     // Check for control-flow out of the current function (which might
    245     // indirectly execute SSE instructions).
    246     if (!isControlFlow)
    247       continue;
    248 
    249     BBHasCall = true;
    250 
    251     // The VZEROUPPER instruction resets the upper 128 bits of all Intel AVX
    252     // registers. This instruction has zero latency. In addition, the processor
    253     // changes back to Clean state, after which execution of Intel SSE
    254     // instructions or Intel AVX instructions has no transition penalty. Add
    255     // the VZEROUPPER instruction before any function call/return that might
    256     // execute SSE code.
    257     // FIXME: In some cases, we may want to move the VZEROUPPER into a
    258     // predecessor block.
    259     if (CurState == ST_DIRTY) {
    260       // Only insert the VZEROUPPER in case the entry state isn't unknown.
    261       // When unknown, only compute the information within the block to have
    262       // it available in the exit if possible, but don't change the block.
    263       if (EntryState != ST_UNKNOWN) {
    264         BuildMI(BB, I, dl, TII->get(X86::VZEROUPPER));
    265         ++NumVZU;
    266       }
    267 
    268       // After the inserted VZEROUPPER the state becomes clean again, but
    269       // other YMM may appear before other subsequent calls or even before
    270       // the end of the BB.
    271       CurState = ST_CLEAN;
    272     }
    273   }
    274 
    275   DEBUG(dbgs() << "MBB #" << BBNum
    276                << ", current state: " << CurState << '\n');
    277 
    278   // A BB can only be considered solved when we both have done all the
    279   // necessary transformations, and have computed the exit state.  This happens
    280   // in two cases:
    281   //  1) We know the entry state: this immediately implies the exit state and
    282   //     all the necessary transformations.
    283   //  2) There are no calls, and and a non-call instruction marks this block:
    284   //     no transformations are necessary, and we know the exit state.
    285   if (EntryState != ST_UNKNOWN || (!BBHasCall && CurState != ST_UNKNOWN))
    286     BBSolved[BBNum] = true;
    287 
    288   if (CurState != BBState[BBNum])
    289     Changed = true;
    290 
    291   BBState[BBNum] = CurState;
    292   return Changed;
    293 }
    294