Home | History | Annotate | Download | only in R600 
      1 //===-- R600MachineScheduler.cpp - R600 Scheduler Interface -*- C++ -*-----===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 /// \file
     11 /// \brief R600 Machine Scheduler interface
     12 //
     13 //===----------------------------------------------------------------------===//
     14 
     15 #include "R600MachineScheduler.h"
     16 #include "AMDGPUSubtarget.h"
     17 #include "llvm/CodeGen/LiveIntervalAnalysis.h"
     18 #include "llvm/CodeGen/MachineRegisterInfo.h"
     19 #include "llvm/Pass.h"
     20 #include "llvm/PassManager.h"
     21 #include "llvm/Support/raw_ostream.h"
     22 
     23 using namespace llvm;
     24 
     25 #define DEBUG_TYPE "misched"
     26 
     27 void R600SchedStrategy::initialize(ScheduleDAGMI *dag) {
     28   assert(dag->hasVRegLiveness() && "R600SchedStrategy needs vreg liveness");
     29   DAG = static_cast<ScheduleDAGMILive*>(dag);
     30   TII = static_cast<const R600InstrInfo*>(DAG->TII);
     31   TRI = static_cast<const R600RegisterInfo*>(DAG->TRI);
     32   VLIW5 = !DAG->MF.getTarget().getSubtarget<AMDGPUSubtarget>().hasCaymanISA();
     33   MRI = &DAG->MRI;
     34   CurInstKind = IDOther;
     35   CurEmitted = 0;
     36   OccupedSlotsMask = 31;
     37   InstKindLimit[IDAlu] = TII->getMaxAlusPerClause();
     38   InstKindLimit[IDOther] = 32;
     39 
     40   const AMDGPUSubtarget &ST = DAG->TM.getSubtarget<AMDGPUSubtarget>();
     41   InstKindLimit[IDFetch] = ST.getTexVTXClauseSize();
     42   AluInstCount = 0;
     43   FetchInstCount = 0;
     44 }
     45 
     46 void R600SchedStrategy::MoveUnits(std::vector<SUnit *> &QSrc,
     47                                   std::vector<SUnit *> &QDst)
     48 {
     49   QDst.insert(QDst.end(), QSrc.begin(), QSrc.end());
     50   QSrc.clear();
     51 }
     52 
     53 static
     54 unsigned getWFCountLimitedByGPR(unsigned GPRCount) {
     55   assert (GPRCount && "GPRCount cannot be 0");
     56   return 248 / GPRCount;
     57 }
     58 
     59 SUnit* R600SchedStrategy::pickNode(bool &IsTopNode) {
     60   SUnit *SU = nullptr;
     61   NextInstKind = IDOther;
     62 
     63   IsTopNode = false;
     64 
     65   // check if we might want to switch current clause type
     66   bool AllowSwitchToAlu = (CurEmitted >= InstKindLimit[CurInstKind]) ||
     67       (Available[CurInstKind].empty());
     68   bool AllowSwitchFromAlu = (CurEmitted >= InstKindLimit[CurInstKind]) &&
     69       (!Available[IDFetch].empty() || !Available[IDOther].empty());
     70 
     71   if (CurInstKind == IDAlu && !Available[IDFetch].empty()) {
     72     // We use the heuristic provided by AMD Accelerated Parallel Processing
     73     // OpenCL Programming Guide :
     74     // The approx. number of WF that allows TEX inst to hide ALU inst is :
     75     // 500 (cycles for TEX) / (AluFetchRatio * 8 (cycles for ALU))
     76     float ALUFetchRationEstimate =
     77         (AluInstCount + AvailablesAluCount() + Pending[IDAlu].size()) /
     78         (FetchInstCount + Available[IDFetch].size());
     79     unsigned NeededWF = 62.5f / ALUFetchRationEstimate;
     80     DEBUG( dbgs() << NeededWF << " approx. Wavefronts Required\n" );
     81     // We assume the local GPR requirements to be "dominated" by the requirement
     82     // of the TEX clause (which consumes 128 bits regs) ; ALU inst before and
     83     // after TEX are indeed likely to consume or generate values from/for the
     84     // TEX clause.
     85     // Available[IDFetch].size() * 2 : GPRs required in the Fetch clause
     86     // We assume that fetch instructions are either TnXYZW = TEX TnXYZW (need
     87     // one GPR) or TmXYZW = TnXYZW (need 2 GPR).
     88     // (TODO : use RegisterPressure)
     89     // If we are going too use too many GPR, we flush Fetch instruction to lower
     90     // register pressure on 128 bits regs.
     91     unsigned NearRegisterRequirement = 2 * Available[IDFetch].size();
     92     if (NeededWF > getWFCountLimitedByGPR(NearRegisterRequirement))
     93       AllowSwitchFromAlu = true;
     94   }
     95 
     96   if (!SU && ((AllowSwitchToAlu && CurInstKind != IDAlu) ||
     97       (!AllowSwitchFromAlu && CurInstKind == IDAlu))) {
     98     // try to pick ALU
     99     SU = pickAlu();
    100     if (!SU && !PhysicalRegCopy.empty()) {
    101       SU = PhysicalRegCopy.front();
    102       PhysicalRegCopy.erase(PhysicalRegCopy.begin());
    103     }
    104     if (SU) {
    105       if (CurEmitted >= InstKindLimit[IDAlu])
    106         CurEmitted = 0;
    107       NextInstKind = IDAlu;
    108     }
    109   }
    110 
    111   if (!SU) {
    112     // try to pick FETCH
    113     SU = pickOther(IDFetch);
    114     if (SU)
    115       NextInstKind = IDFetch;
    116   }
    117 
    118   // try to pick other
    119   if (!SU) {
    120     SU = pickOther(IDOther);
    121     if (SU)
    122       NextInstKind = IDOther;
    123   }
    124 
    125   DEBUG(
    126       if (SU) {
    127         dbgs() << " ** Pick node **\n";
    128         SU->dump(DAG);
    129       } else {
    130         dbgs() << "NO NODE \n";
    131         for (unsigned i = 0; i < DAG->SUnits.size(); i++) {
    132           const SUnit &S = DAG->SUnits[i];
    133           if (!S.isScheduled)
    134             S.dump(DAG);
    135         }
    136       }
    137   );
    138 
    139   return SU;
    140 }
    141 
    142 void R600SchedStrategy::schedNode(SUnit *SU, bool IsTopNode) {
    143   if (NextInstKind != CurInstKind) {
    144     DEBUG(dbgs() << "Instruction Type Switch\n");
    145     if (NextInstKind != IDAlu)
    146       OccupedSlotsMask |= 31;
    147     CurEmitted = 0;
    148     CurInstKind = NextInstKind;
    149   }
    150 
    151   if (CurInstKind == IDAlu) {
    152     AluInstCount ++;
    153     switch (getAluKind(SU)) {
    154     case AluT_XYZW:
    155       CurEmitted += 4;
    156       break;
    157     case AluDiscarded:
    158       break;
    159     default: {
    160       ++CurEmitted;
    161       for (MachineInstr::mop_iterator It = SU->getInstr()->operands_begin(),
    162           E = SU->getInstr()->operands_end(); It != E; ++It) {
    163         MachineOperand &MO = *It;
    164         if (MO.isReg() && MO.getReg() == AMDGPU::ALU_LITERAL_X)
    165           ++CurEmitted;
    166       }
    167     }
    168     }
    169   } else {
    170     ++CurEmitted;
    171   }
    172 
    173 
    174   DEBUG(dbgs() << CurEmitted << " Instructions Emitted in this clause\n");
    175 
    176   if (CurInstKind != IDFetch) {
    177     MoveUnits(Pending[IDFetch], Available[IDFetch]);
    178   } else
    179     FetchInstCount++;
    180 }
    181 
    182 static bool
    183 isPhysicalRegCopy(MachineInstr *MI) {
    184   if (MI->getOpcode() != AMDGPU::COPY)
    185     return false;
    186 
    187   return !TargetRegisterInfo::isVirtualRegister(MI->getOperand(1).getReg());
    188 }
    189 
    190 void R600SchedStrategy::releaseTopNode(SUnit *SU) {
    191   DEBUG(dbgs() << "Top Releasing ";SU->dump(DAG););
    192 }
    193 
    194 void R600SchedStrategy::releaseBottomNode(SUnit *SU) {
    195   DEBUG(dbgs() << "Bottom Releasing ";SU->dump(DAG););
    196   if (isPhysicalRegCopy(SU->getInstr())) {
    197     PhysicalRegCopy.push_back(SU);
    198     return;
    199   }
    200 
    201   int IK = getInstKind(SU);
    202 
    203   // There is no export clause, we can schedule one as soon as its ready
    204   if (IK == IDOther)
    205     Available[IDOther].push_back(SU);
    206   else
    207     Pending[IK].push_back(SU);
    208 
    209 }
    210 
    211 bool R600SchedStrategy::regBelongsToClass(unsigned Reg,
    212                                           const TargetRegisterClass *RC) const {
    213   if (!TargetRegisterInfo::isVirtualRegister(Reg)) {
    214     return RC->contains(Reg);
    215   } else {
    216     return MRI->getRegClass(Reg) == RC;
    217   }
    218 }
    219 
    220 R600SchedStrategy::AluKind R600SchedStrategy::getAluKind(SUnit *SU) const {
    221   MachineInstr *MI = SU->getInstr();
    222 
    223   if (TII->isTransOnly(MI))
    224     return AluTrans;
    225 
    226     switch (MI->getOpcode()) {
    227     case AMDGPU::PRED_X:
    228       return AluPredX;
    229     case AMDGPU::INTERP_PAIR_XY:
    230     case AMDGPU::INTERP_PAIR_ZW:
    231     case AMDGPU::INTERP_VEC_LOAD:
    232     case AMDGPU::DOT_4:
    233       return AluT_XYZW;
    234     case AMDGPU::COPY:
    235       if (MI->getOperand(1).isUndef()) {
    236         // MI will become a KILL, don't considers it in scheduling
    237         return AluDiscarded;
    238       }
    239     default:
    240       break;
    241     }
    242 
    243     // Does the instruction take a whole IG ?
    244     // XXX: Is it possible to add a helper function in R600InstrInfo that can
    245     // be used here and in R600PacketizerList::isSoloInstruction() ?
    246     if(TII->isVector(*MI) ||
    247         TII->isCubeOp(MI->getOpcode()) ||
    248         TII->isReductionOp(MI->getOpcode()) ||
    249         MI->getOpcode() == AMDGPU::GROUP_BARRIER) {
    250       return AluT_XYZW;
    251     }
    252 
    253     if (TII->isLDSInstr(MI->getOpcode())) {
    254       return AluT_X;
    255     }
    256 
    257     // Is the result already assigned to a channel ?
    258     unsigned DestSubReg = MI->getOperand(0).getSubReg();
    259     switch (DestSubReg) {
    260     case AMDGPU::sub0:
    261       return AluT_X;
    262     case AMDGPU::sub1:
    263       return AluT_Y;
    264     case AMDGPU::sub2:
    265       return AluT_Z;
    266     case AMDGPU::sub3:
    267       return AluT_W;
    268     default:
    269       break;
    270     }
    271 
    272     // Is the result already member of a X/Y/Z/W class ?
    273     unsigned DestReg = MI->getOperand(0).getReg();
    274     if (regBelongsToClass(DestReg, &AMDGPU::R600_TReg32_XRegClass) ||
    275         regBelongsToClass(DestReg, &AMDGPU::R600_AddrRegClass))
    276       return AluT_X;
    277     if (regBelongsToClass(DestReg, &AMDGPU::R600_TReg32_YRegClass))
    278       return AluT_Y;
    279     if (regBelongsToClass(DestReg, &AMDGPU::R600_TReg32_ZRegClass))
    280       return AluT_Z;
    281     if (regBelongsToClass(DestReg, &AMDGPU::R600_TReg32_WRegClass))
    282       return AluT_W;
    283     if (regBelongsToClass(DestReg, &AMDGPU::R600_Reg128RegClass))
    284       return AluT_XYZW;
    285 
    286     // LDS src registers cannot be used in the Trans slot.
    287     if (TII->readsLDSSrcReg(MI))
    288       return AluT_XYZW;
    289 
    290     return AluAny;
    291 
    292 }
    293 
    294 int R600SchedStrategy::getInstKind(SUnit* SU) {
    295   int Opcode = SU->getInstr()->getOpcode();
    296 
    297   if (TII->usesTextureCache(Opcode) || TII->usesVertexCache(Opcode))
    298     return IDFetch;
    299 
    300   if (TII->isALUInstr(Opcode)) {
    301     return IDAlu;
    302   }
    303 
    304   switch (Opcode) {
    305   case AMDGPU::PRED_X:
    306   case AMDGPU::COPY:
    307   case AMDGPU::CONST_COPY:
    308   case AMDGPU::INTERP_PAIR_XY:
    309   case AMDGPU::INTERP_PAIR_ZW:
    310   case AMDGPU::INTERP_VEC_LOAD:
    311   case AMDGPU::DOT_4:
    312     return IDAlu;
    313   default:
    314     return IDOther;
    315   }
    316 }
    317 
    318 SUnit *R600SchedStrategy::PopInst(std::vector<SUnit *> &Q, bool AnyALU) {
    319   if (Q.empty())
    320     return nullptr;
    321   for (std::vector<SUnit *>::reverse_iterator It = Q.rbegin(), E = Q.rend();
    322       It != E; ++It) {
    323     SUnit *SU = *It;
    324     InstructionsGroupCandidate.push_back(SU->getInstr());
    325     if (TII->fitsConstReadLimitations(InstructionsGroupCandidate)
    326         && (!AnyALU || !TII->isVectorOnly(SU->getInstr()))
    327     ) {
    328       InstructionsGroupCandidate.pop_back();
    329       Q.erase((It + 1).base());
    330       return SU;
    331     } else {
    332       InstructionsGroupCandidate.pop_back();
    333     }
    334   }
    335   return nullptr;
    336 }
    337 
    338 void R600SchedStrategy::LoadAlu() {
    339   std::vector<SUnit *> &QSrc = Pending[IDAlu];
    340   for (unsigned i = 0, e = QSrc.size(); i < e; ++i) {
    341     AluKind AK = getAluKind(QSrc[i]);
    342     AvailableAlus[AK].push_back(QSrc[i]);
    343   }
    344   QSrc.clear();
    345 }
    346 
    347 void R600SchedStrategy::PrepareNextSlot() {
    348   DEBUG(dbgs() << "New Slot\n");
    349   assert (OccupedSlotsMask && "Slot wasn't filled");
    350   OccupedSlotsMask = 0;
    351 //  if (HwGen == AMDGPUSubtarget::NORTHERN_ISLANDS)
    352 //    OccupedSlotsMask |= 16;
    353   InstructionsGroupCandidate.clear();
    354   LoadAlu();
    355 }
    356 
    357 void R600SchedStrategy::AssignSlot(MachineInstr* MI, unsigned Slot) {
    358   int DstIndex = TII->getOperandIdx(MI->getOpcode(), AMDGPU::OpName::dst);
    359   if (DstIndex == -1) {
    360     return;
    361   }
    362   unsigned DestReg = MI->getOperand(DstIndex).getReg();
    363   // PressureRegister crashes if an operand is def and used in the same inst
    364   // and we try to constraint its regclass
    365   for (MachineInstr::mop_iterator It = MI->operands_begin(),
    366       E = MI->operands_end(); It != E; ++It) {
    367     MachineOperand &MO = *It;
    368     if (MO.isReg() && !MO.isDef() &&
    369         MO.getReg() == DestReg)
    370       return;
    371   }
    372   // Constrains the regclass of DestReg to assign it to Slot
    373   switch (Slot) {
    374   case 0:
    375     MRI->constrainRegClass(DestReg, &AMDGPU::R600_TReg32_XRegClass);
    376     break;
    377   case 1:
    378     MRI->constrainRegClass(DestReg, &AMDGPU::R600_TReg32_YRegClass);
    379     break;
    380   case 2:
    381     MRI->constrainRegClass(DestReg, &AMDGPU::R600_TReg32_ZRegClass);
    382     break;
    383   case 3:
    384     MRI->constrainRegClass(DestReg, &AMDGPU::R600_TReg32_WRegClass);
    385     break;
    386   }
    387 }
    388 
    389 SUnit *R600SchedStrategy::AttemptFillSlot(unsigned Slot, bool AnyAlu) {
    390   static const AluKind IndexToID[] = {AluT_X, AluT_Y, AluT_Z, AluT_W};
    391   SUnit *SlotedSU = PopInst(AvailableAlus[IndexToID[Slot]], AnyAlu);
    392   if (SlotedSU)
    393     return SlotedSU;
    394   SUnit *UnslotedSU = PopInst(AvailableAlus[AluAny], AnyAlu);
    395   if (UnslotedSU)
    396     AssignSlot(UnslotedSU->getInstr(), Slot);
    397   return UnslotedSU;
    398 }
    399 
    400 unsigned R600SchedStrategy::AvailablesAluCount() const {
    401   return AvailableAlus[AluAny].size() + AvailableAlus[AluT_XYZW].size() +
    402       AvailableAlus[AluT_X].size() + AvailableAlus[AluT_Y].size() +
    403       AvailableAlus[AluT_Z].size() + AvailableAlus[AluT_W].size() +
    404       AvailableAlus[AluTrans].size() + AvailableAlus[AluDiscarded].size() +
    405       AvailableAlus[AluPredX].size();
    406 }
    407 
    408 SUnit* R600SchedStrategy::pickAlu() {
    409   while (AvailablesAluCount() || !Pending[IDAlu].empty()) {
    410     if (!OccupedSlotsMask) {
    411       // Bottom up scheduling : predX must comes first
    412       if (!AvailableAlus[AluPredX].empty()) {
    413         OccupedSlotsMask |= 31;
    414         return PopInst(AvailableAlus[AluPredX], false);
    415       }
    416       // Flush physical reg copies (RA will discard them)
    417       if (!AvailableAlus[AluDiscarded].empty()) {
    418         OccupedSlotsMask |= 31;
    419         return PopInst(AvailableAlus[AluDiscarded], false);
    420       }
    421       // If there is a T_XYZW alu available, use it
    422       if (!AvailableAlus[AluT_XYZW].empty()) {
    423         OccupedSlotsMask |= 15;
    424         return PopInst(AvailableAlus[AluT_XYZW], false);
    425       }
    426     }
    427     bool TransSlotOccuped = OccupedSlotsMask & 16;
    428     if (!TransSlotOccuped && VLIW5) {
    429       if (!AvailableAlus[AluTrans].empty()) {
    430         OccupedSlotsMask |= 16;
    431         return PopInst(AvailableAlus[AluTrans], false);
    432       }
    433       SUnit *SU = AttemptFillSlot(3, true);
    434       if (SU) {
    435         OccupedSlotsMask |= 16;
    436         return SU;
    437       }
    438     }
    439     for (int Chan = 3; Chan > -1; --Chan) {
    440       bool isOccupied = OccupedSlotsMask & (1 << Chan);
    441       if (!isOccupied) {
    442         SUnit *SU = AttemptFillSlot(Chan, false);
    443         if (SU) {
    444           OccupedSlotsMask |= (1 << Chan);
    445           InstructionsGroupCandidate.push_back(SU->getInstr());
    446           return SU;
    447         }
    448       }
    449     }
    450     PrepareNextSlot();
    451   }
    452   return nullptr;
    453 }
    454 
    455 SUnit* R600SchedStrategy::pickOther(int QID) {
    456   SUnit *SU = nullptr;
    457   std::vector<SUnit *> &AQ = Available[QID];
    458 
    459   if (AQ.empty()) {
    460     MoveUnits(Pending[QID], AQ);
    461   }
    462   if (!AQ.empty()) {
    463     SU = AQ.back();
    464     AQ.resize(AQ.size() - 1);
    465   }
    466   return SU;
    467 }
    468