Home | History | Annotate | Download | only in AMDGPU 
      1 //===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //==-----------------------------------------------------------------------===//
      9 //
     10 /// \file
     11 /// \brief Defines an instruction selector for the AMDGPU target.
     12 //
     13 //===----------------------------------------------------------------------===//
     14 
     15 #include "AMDGPUInstrInfo.h"
     16 #include "AMDGPUIntrinsicInfo.h"
     17 #include "AMDGPUISelLowering.h" // For AMDGPUISD
     18 #include "AMDGPUSubtarget.h"
     19 #include "SIISelLowering.h"
     20 #include "SIMachineFunctionInfo.h"
     21 #include "llvm/Analysis/ValueTracking.h"
     22 #include "llvm/CodeGen/FunctionLoweringInfo.h"
     23 #include "llvm/CodeGen/MachineFrameInfo.h"
     24 #include "llvm/CodeGen/PseudoSourceValue.h"
     25 #include "llvm/CodeGen/SelectionDAG.h"
     26 #include "llvm/CodeGen/SelectionDAGISel.h"
     27 #include "llvm/IR/DiagnosticInfo.h"
     28 
     29 using namespace llvm;
     30 
     31 namespace llvm {
     32 class R600InstrInfo;
     33 }
     34 
     35 //===----------------------------------------------------------------------===//
     36 // Instruction Selector Implementation
     37 //===----------------------------------------------------------------------===//
     38 
     39 namespace {
     40 
     41 static bool isCBranchSCC(const SDNode *N) {
     42   assert(N->getOpcode() == ISD::BRCOND);
     43   if (!N->hasOneUse())
     44     return false;
     45 
     46   SDValue Cond = N->getOperand(1);
     47   if (Cond.getOpcode() == ISD::CopyToReg)
     48     Cond = Cond.getOperand(2);
     49   return Cond.getOpcode() == ISD::SETCC &&
     50          Cond.getOperand(0).getValueType() == MVT::i32 && Cond.hasOneUse();
     51 }
     52 
     53 /// AMDGPU specific code to select AMDGPU machine instructions for
     54 /// SelectionDAG operations.
     55 class AMDGPUDAGToDAGISel : public SelectionDAGISel {
     56   // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can
     57   // make the right decision when generating code for different targets.
     58   const AMDGPUSubtarget *Subtarget;
     59 
     60 public:
     61   AMDGPUDAGToDAGISel(TargetMachine &TM);
     62   virtual ~AMDGPUDAGToDAGISel();
     63   bool runOnMachineFunction(MachineFunction &MF) override;
     64   void Select(SDNode *N) override;
     65   const char *getPassName() const override;
     66   void PreprocessISelDAG() override;
     67   void PostprocessISelDAG() override;
     68 
     69 private:
     70   bool isInlineImmediate(SDNode *N) const;
     71   bool FoldOperand(SDValue &Src, SDValue &Sel, SDValue &Neg, SDValue &Abs,
     72                    const R600InstrInfo *TII);
     73   bool FoldOperands(unsigned, const R600InstrInfo *, std::vector<SDValue> &);
     74   bool FoldDotOperands(unsigned, const R600InstrInfo *, std::vector<SDValue> &);
     75 
     76   bool isConstantLoad(const MemSDNode *N, int cbID) const;
     77   bool isUniformBr(const SDNode *N) const;
     78 
     79   SDNode *glueCopyToM0(SDNode *N) const;
     80 
     81   const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
     82   bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr);
     83   bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg,
     84                                        SDValue& Offset);
     85   bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
     86   bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
     87   bool isDSOffsetLegal(const SDValue &Base, unsigned Offset,
     88                        unsigned OffsetBits) const;
     89   bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
     90   bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
     91                                  SDValue &Offset1) const;
     92   bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
     93                    SDValue &SOffset, SDValue &Offset, SDValue &Offen,
     94                    SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC,
     95                    SDValue &TFE) const;
     96   bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
     97                          SDValue &SOffset, SDValue &Offset, SDValue &GLC,
     98                          SDValue &SLC, SDValue &TFE) const;
     99   bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
    100                          SDValue &VAddr, SDValue &SOffset, SDValue &Offset,
    101                          SDValue &SLC) const;
    102   bool SelectMUBUFScratch(SDValue Addr, SDValue &RSrc, SDValue &VAddr,
    103                           SDValue &SOffset, SDValue &ImmOffset) const;
    104   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset,
    105                          SDValue &Offset, SDValue &GLC, SDValue &SLC,
    106                          SDValue &TFE) const;
    107   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
    108                          SDValue &Offset, SDValue &SLC) const;
    109   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
    110                          SDValue &Offset) const;
    111   bool SelectMUBUFConstant(SDValue Constant,
    112                            SDValue &SOffset,
    113                            SDValue &ImmOffset) const;
    114   bool SelectMUBUFIntrinsicOffset(SDValue Offset, SDValue &SOffset,
    115                                   SDValue &ImmOffset) const;
    116   bool SelectMUBUFIntrinsicVOffset(SDValue Offset, SDValue &SOffset,
    117                                    SDValue &ImmOffset, SDValue &VOffset) const;
    118 
    119   bool SelectFlat(SDValue Addr, SDValue &VAddr,
    120                   SDValue &SLC, SDValue &TFE) const;
    121 
    122   bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset,
    123                         bool &Imm) const;
    124   bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset,
    125                   bool &Imm) const;
    126   bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
    127   bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
    128   bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
    129   bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const;
    130   bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const;
    131   bool SelectSMRDBufferSgpr(SDValue Addr, SDValue &Offset) const;
    132   bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
    133   bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
    134   bool SelectVOP3NoMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
    135   bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
    136                        SDValue &Clamp, SDValue &Omod) const;
    137   bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
    138                          SDValue &Clamp, SDValue &Omod) const;
    139 
    140   bool SelectVOP3Mods0Clamp(SDValue In, SDValue &Src, SDValue &SrcMods,
    141                             SDValue &Omod) const;
    142   bool SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src, SDValue &SrcMods,
    143                                  SDValue &Clamp,
    144                                  SDValue &Omod) const;
    145 
    146   void SelectADD_SUB_I64(SDNode *N);
    147   void SelectDIV_SCALE(SDNode *N);
    148 
    149   SDNode *getS_BFE(unsigned Opcode, const SDLoc &DL, SDValue Val,
    150                    uint32_t Offset, uint32_t Width);
    151   void SelectS_BFEFromShifts(SDNode *N);
    152   void SelectS_BFE(SDNode *N);
    153   void SelectBRCOND(SDNode *N);
    154   void SelectATOMIC_CMP_SWAP(SDNode *N);
    155 
    156   // Include the pieces autogenerated from the target description.
    157 #include "AMDGPUGenDAGISel.inc"
    158 };
    159 }  // end anonymous namespace
    160 
    161 /// \brief This pass converts a legalized DAG into a AMDGPU-specific
    162 // DAG, ready for instruction scheduling.
    163 FunctionPass *llvm::createAMDGPUISelDag(TargetMachine &TM) {
    164   return new AMDGPUDAGToDAGISel(TM);
    165 }
    166 
    167 AMDGPUDAGToDAGISel::AMDGPUDAGToDAGISel(TargetMachine &TM)
    168     : SelectionDAGISel(TM) {}
    169 
    170 bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
    171   Subtarget = &MF.getSubtarget<AMDGPUSubtarget>();
    172   return SelectionDAGISel::runOnMachineFunction(MF);
    173 }
    174 
    175 AMDGPUDAGToDAGISel::~AMDGPUDAGToDAGISel() {
    176 }
    177 
    178 bool AMDGPUDAGToDAGISel::isInlineImmediate(SDNode *N) const {
    179   const SITargetLowering *TL
    180       = static_cast<const SITargetLowering *>(getTargetLowering());
    181   return TL->analyzeImmediate(N) == 0;
    182 }
    183 
    184 /// \brief Determine the register class for \p OpNo
    185 /// \returns The register class of the virtual register that will be used for
    186 /// the given operand number \OpNo or NULL if the register class cannot be
    187 /// determined.
    188 const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N,
    189                                                           unsigned OpNo) const {
    190   if (!N->isMachineOpcode())
    191     return nullptr;
    192 
    193   switch (N->getMachineOpcode()) {
    194   default: {
    195     const MCInstrDesc &Desc =
    196         Subtarget->getInstrInfo()->get(N->getMachineOpcode());
    197     unsigned OpIdx = Desc.getNumDefs() + OpNo;
    198     if (OpIdx >= Desc.getNumOperands())
    199       return nullptr;
    200     int RegClass = Desc.OpInfo[OpIdx].RegClass;
    201     if (RegClass == -1)
    202       return nullptr;
    203 
    204     return Subtarget->getRegisterInfo()->getRegClass(RegClass);
    205   }
    206   case AMDGPU::REG_SEQUENCE: {
    207     unsigned RCID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
    208     const TargetRegisterClass *SuperRC =
    209         Subtarget->getRegisterInfo()->getRegClass(RCID);
    210 
    211     SDValue SubRegOp = N->getOperand(OpNo + 1);
    212     unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue();
    213     return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC,
    214                                                               SubRegIdx);
    215   }
    216   }
    217 }
    218 
    219 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N) const {
    220   if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS ||
    221       cast<MemSDNode>(N)->getAddressSpace() != AMDGPUAS::LOCAL_ADDRESS)
    222     return N;
    223 
    224   const SITargetLowering& Lowering =
    225       *static_cast<const SITargetLowering*>(getTargetLowering());
    226 
    227   // Write max value to m0 before each load operation
    228 
    229   SDValue M0 = Lowering.copyToM0(*CurDAG, CurDAG->getEntryNode(), SDLoc(N),
    230                                  CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32));
    231 
    232   SDValue Glue = M0.getValue(1);
    233 
    234   SmallVector <SDValue, 8> Ops;
    235   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
    236      Ops.push_back(N->getOperand(i));
    237   }
    238   Ops.push_back(Glue);
    239   CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops);
    240 
    241   return N;
    242 }
    243 
    244 static unsigned selectSGPRVectorRegClassID(unsigned NumVectorElts) {
    245   switch (NumVectorElts) {
    246   case 1:
    247     return AMDGPU::SReg_32RegClassID;
    248   case 2:
    249     return AMDGPU::SReg_64RegClassID;
    250   case 4:
    251     return AMDGPU::SReg_128RegClassID;
    252   case 8:
    253     return AMDGPU::SReg_256RegClassID;
    254   case 16:
    255     return AMDGPU::SReg_512RegClassID;
    256   }
    257 
    258   llvm_unreachable("invalid vector size");
    259 }
    260 
    261 void AMDGPUDAGToDAGISel::Select(SDNode *N) {
    262   unsigned int Opc = N->getOpcode();
    263   if (N->isMachineOpcode()) {
    264     N->setNodeId(-1);
    265     return;   // Already selected.
    266   }
    267 
    268   if (isa<AtomicSDNode>(N) ||
    269       (Opc == AMDGPUISD::ATOMIC_INC || Opc == AMDGPUISD::ATOMIC_DEC))
    270     N = glueCopyToM0(N);
    271 
    272   switch (Opc) {
    273   default: break;
    274   // We are selecting i64 ADD here instead of custom lower it during
    275   // DAG legalization, so we can fold some i64 ADDs used for address
    276   // calculation into the LOAD and STORE instructions.
    277   case ISD::ADD:
    278   case ISD::SUB: {
    279     if (N->getValueType(0) != MVT::i64 ||
    280         Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS)
    281       break;
    282 
    283     SelectADD_SUB_I64(N);
    284     return;
    285   }
    286   case ISD::SCALAR_TO_VECTOR:
    287   case AMDGPUISD::BUILD_VERTICAL_VECTOR:
    288   case ISD::BUILD_VECTOR: {
    289     unsigned RegClassID;
    290     const AMDGPURegisterInfo *TRI = Subtarget->getRegisterInfo();
    291     EVT VT = N->getValueType(0);
    292     unsigned NumVectorElts = VT.getVectorNumElements();
    293     EVT EltVT = VT.getVectorElementType();
    294     assert(EltVT.bitsEq(MVT::i32));
    295     if (Subtarget->getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS) {
    296       RegClassID = selectSGPRVectorRegClassID(NumVectorElts);
    297     } else {
    298       // BUILD_VECTOR was lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG
    299       // that adds a 128 bits reg copy when going through TwoAddressInstructions
    300       // pass. We want to avoid 128 bits copies as much as possible because they
    301       // can't be bundled by our scheduler.
    302       switch(NumVectorElts) {
    303       case 2: RegClassID = AMDGPU::R600_Reg64RegClassID; break;
    304       case 4:
    305         if (Opc == AMDGPUISD::BUILD_VERTICAL_VECTOR)
    306           RegClassID = AMDGPU::R600_Reg128VerticalRegClassID;
    307         else
    308           RegClassID = AMDGPU::R600_Reg128RegClassID;
    309         break;
    310       default: llvm_unreachable("Do not know how to lower this BUILD_VECTOR");
    311       }
    312     }
    313 
    314     SDLoc DL(N);
    315     SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
    316 
    317     if (NumVectorElts == 1) {
    318       CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT, N->getOperand(0),
    319                            RegClass);
    320       return;
    321     }
    322 
    323     assert(NumVectorElts <= 16 && "Vectors with more than 16 elements not "
    324                                   "supported yet");
    325     // 16 = Max Num Vector Elements
    326     // 2 = 2 REG_SEQUENCE operands per element (value, subreg index)
    327     // 1 = Vector Register Class
    328     SmallVector<SDValue, 16 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1);
    329 
    330     RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
    331     bool IsRegSeq = true;
    332     unsigned NOps = N->getNumOperands();
    333     for (unsigned i = 0; i < NOps; i++) {
    334       // XXX: Why is this here?
    335       if (isa<RegisterSDNode>(N->getOperand(i))) {
    336         IsRegSeq = false;
    337         break;
    338       }
    339       RegSeqArgs[1 + (2 * i)] = N->getOperand(i);
    340       RegSeqArgs[1 + (2 * i) + 1] =
    341               CurDAG->getTargetConstant(TRI->getSubRegFromChannel(i), DL,
    342                                         MVT::i32);
    343     }
    344 
    345     if (NOps != NumVectorElts) {
    346       // Fill in the missing undef elements if this was a scalar_to_vector.
    347       assert(Opc == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts);
    348 
    349       MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
    350                                                      DL, EltVT);
    351       for (unsigned i = NOps; i < NumVectorElts; ++i) {
    352         RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0);
    353         RegSeqArgs[1 + (2 * i) + 1] =
    354           CurDAG->getTargetConstant(TRI->getSubRegFromChannel(i), DL, MVT::i32);
    355       }
    356     }
    357 
    358     if (!IsRegSeq)
    359       break;
    360     CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(), RegSeqArgs);
    361     return;
    362   }
    363   case ISD::BUILD_PAIR: {
    364     SDValue RC, SubReg0, SubReg1;
    365     if (Subtarget->getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS) {
    366       break;
    367     }
    368     SDLoc DL(N);
    369     if (N->getValueType(0) == MVT::i128) {
    370       RC = CurDAG->getTargetConstant(AMDGPU::SReg_128RegClassID, DL, MVT::i32);
    371       SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32);
    372       SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32);
    373     } else if (N->getValueType(0) == MVT::i64) {
    374       RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32);
    375       SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
    376       SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
    377     } else {
    378       llvm_unreachable("Unhandled value type for BUILD_PAIR");
    379     }
    380     const SDValue Ops[] = { RC, N->getOperand(0), SubReg0,
    381                             N->getOperand(1), SubReg1 };
    382     ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
    383                                           N->getValueType(0), Ops));
    384     return;
    385   }
    386 
    387   case ISD::Constant:
    388   case ISD::ConstantFP: {
    389     if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS ||
    390         N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N))
    391       break;
    392 
    393     uint64_t Imm;
    394     if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(N))
    395       Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue();
    396     else {
    397       ConstantSDNode *C = cast<ConstantSDNode>(N);
    398       Imm = C->getZExtValue();
    399     }
    400 
    401     SDLoc DL(N);
    402     SDNode *Lo = CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
    403                                 CurDAG->getConstant(Imm & 0xFFFFFFFF, DL,
    404                                                     MVT::i32));
    405     SDNode *Hi = CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
    406                                 CurDAG->getConstant(Imm >> 32, DL, MVT::i32));
    407     const SDValue Ops[] = {
    408       CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
    409       SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
    410       SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)
    411     };
    412 
    413     ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
    414                                           N->getValueType(0), Ops));
    415     return;
    416   }
    417   case ISD::LOAD:
    418   case ISD::STORE: {
    419     N = glueCopyToM0(N);
    420     break;
    421   }
    422 
    423   case AMDGPUISD::BFE_I32:
    424   case AMDGPUISD::BFE_U32: {
    425     if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS)
    426       break;
    427 
    428     // There is a scalar version available, but unlike the vector version which
    429     // has a separate operand for the offset and width, the scalar version packs
    430     // the width and offset into a single operand. Try to move to the scalar
    431     // version if the offsets are constant, so that we can try to keep extended
    432     // loads of kernel arguments in SGPRs.
    433 
    434     // TODO: Technically we could try to pattern match scalar bitshifts of
    435     // dynamic values, but it's probably not useful.
    436     ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
    437     if (!Offset)
    438       break;
    439 
    440     ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
    441     if (!Width)
    442       break;
    443 
    444     bool Signed = Opc == AMDGPUISD::BFE_I32;
    445 
    446     uint32_t OffsetVal = Offset->getZExtValue();
    447     uint32_t WidthVal = Width->getZExtValue();
    448 
    449     ReplaceNode(N, getS_BFE(Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32,
    450                             SDLoc(N), N->getOperand(0), OffsetVal, WidthVal));
    451     return;
    452   }
    453   case AMDGPUISD::DIV_SCALE: {
    454     SelectDIV_SCALE(N);
    455     return;
    456   }
    457   case ISD::CopyToReg: {
    458     const SITargetLowering& Lowering =
    459       *static_cast<const SITargetLowering*>(getTargetLowering());
    460     Lowering.legalizeTargetIndependentNode(N, *CurDAG);
    461     break;
    462   }
    463   case ISD::AND:
    464   case ISD::SRL:
    465   case ISD::SRA:
    466   case ISD::SIGN_EXTEND_INREG:
    467     if (N->getValueType(0) != MVT::i32 ||
    468         Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS)
    469       break;
    470 
    471     SelectS_BFE(N);
    472     return;
    473   case ISD::BRCOND:
    474     SelectBRCOND(N);
    475     return;
    476 
    477   case AMDGPUISD::ATOMIC_CMP_SWAP:
    478     SelectATOMIC_CMP_SWAP(N);
    479     return;
    480   }
    481 
    482   SelectCode(N);
    483 }
    484 
    485 bool AMDGPUDAGToDAGISel::isConstantLoad(const MemSDNode *N, int CbId) const {
    486   if (!N->readMem())
    487     return false;
    488   if (CbId == -1)
    489     return N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS;
    490 
    491   return N->getAddressSpace() == AMDGPUAS::CONSTANT_BUFFER_0 + CbId;
    492 }
    493 
    494 bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const {
    495   const BasicBlock *BB = FuncInfo->MBB->getBasicBlock();
    496   const Instruction *Term = BB->getTerminator();
    497   return Term->getMetadata("amdgpu.uniform") ||
    498          Term->getMetadata("structurizecfg.uniform");
    499 }
    500 
    501 const char *AMDGPUDAGToDAGISel::getPassName() const {
    502   return "AMDGPU DAG->DAG Pattern Instruction Selection";
    503 }
    504 
    505 //===----------------------------------------------------------------------===//
    506 // Complex Patterns
    507 //===----------------------------------------------------------------------===//
    508 
    509 bool AMDGPUDAGToDAGISel::SelectGlobalValueConstantOffset(SDValue Addr,
    510                                                          SDValue& IntPtr) {
    511   if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Addr)) {
    512     IntPtr = CurDAG->getIntPtrConstant(Cst->getZExtValue() / 4, SDLoc(Addr),
    513                                        true);
    514     return true;
    515   }
    516   return false;
    517 }
    518 
    519 bool AMDGPUDAGToDAGISel::SelectGlobalValueVariableOffset(SDValue Addr,
    520     SDValue& BaseReg, SDValue &Offset) {
    521   if (!isa<ConstantSDNode>(Addr)) {
    522     BaseReg = Addr;
    523     Offset = CurDAG->getIntPtrConstant(0, SDLoc(Addr), true);
    524     return true;
    525   }
    526   return false;
    527 }
    528 
    529 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
    530                                            SDValue &Offset) {
    531   ConstantSDNode *IMMOffset;
    532 
    533   if (Addr.getOpcode() == ISD::ADD
    534       && (IMMOffset = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))
    535       && isInt<16>(IMMOffset->getZExtValue())) {
    536 
    537       Base = Addr.getOperand(0);
    538       Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
    539                                          MVT::i32);
    540       return true;
    541   // If the pointer address is constant, we can move it to the offset field.
    542   } else if ((IMMOffset = dyn_cast<ConstantSDNode>(Addr))
    543              && isInt<16>(IMMOffset->getZExtValue())) {
    544     Base = CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
    545                                   SDLoc(CurDAG->getEntryNode()),
    546                                   AMDGPU::ZERO, MVT::i32);
    547     Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
    548                                        MVT::i32);
    549     return true;
    550   }
    551 
    552   // Default case, no offset
    553   Base = Addr;
    554   Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
    555   return true;
    556 }
    557 
    558 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
    559                                             SDValue &Offset) {
    560   ConstantSDNode *C;
    561   SDLoc DL(Addr);
    562 
    563   if ((C = dyn_cast<ConstantSDNode>(Addr))) {
    564     Base = CurDAG->getRegister(AMDGPU::INDIRECT_BASE_ADDR, MVT::i32);
    565     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
    566   } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
    567             (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
    568     Base = Addr.getOperand(0);
    569     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
    570   } else {
    571     Base = Addr;
    572     Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
    573   }
    574 
    575   return true;
    576 }
    577 
    578 void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) {
    579   SDLoc DL(N);
    580   SDValue LHS = N->getOperand(0);
    581   SDValue RHS = N->getOperand(1);
    582 
    583   bool IsAdd = (N->getOpcode() == ISD::ADD);
    584 
    585   SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
    586   SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
    587 
    588   SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
    589                                        DL, MVT::i32, LHS, Sub0);
    590   SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
    591                                        DL, MVT::i32, LHS, Sub1);
    592 
    593   SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
    594                                        DL, MVT::i32, RHS, Sub0);
    595   SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
    596                                        DL, MVT::i32, RHS, Sub1);
    597 
    598   SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue);
    599   SDValue AddLoArgs[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) };
    600 
    601   unsigned Opc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
    602   unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
    603 
    604   SDNode *AddLo = CurDAG->getMachineNode( Opc, DL, VTList, AddLoArgs);
    605   SDValue Carry(AddLo, 1);
    606   SDNode *AddHi
    607     = CurDAG->getMachineNode(CarryOpc, DL, MVT::i32,
    608                              SDValue(Hi0, 0), SDValue(Hi1, 0), Carry);
    609 
    610   SDValue Args[5] = {
    611     CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
    612     SDValue(AddLo,0),
    613     Sub0,
    614     SDValue(AddHi,0),
    615     Sub1,
    616   };
    617   CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, MVT::i64, Args);
    618 }
    619 
    620 // We need to handle this here because tablegen doesn't support matching
    621 // instructions with multiple outputs.
    622 void AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) {
    623   SDLoc SL(N);
    624   EVT VT = N->getValueType(0);
    625 
    626   assert(VT == MVT::f32 || VT == MVT::f64);
    627 
    628   unsigned Opc
    629     = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64 : AMDGPU::V_DIV_SCALE_F32;
    630 
    631   // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp,
    632   // omod
    633   SDValue Ops[8];
    634 
    635   SelectVOP3Mods0(N->getOperand(0), Ops[1], Ops[0], Ops[6], Ops[7]);
    636   SelectVOP3Mods(N->getOperand(1), Ops[3], Ops[2]);
    637   SelectVOP3Mods(N->getOperand(2), Ops[5], Ops[4]);
    638   CurDAG->SelectNodeTo(N, Opc, VT, MVT::i1, Ops);
    639 }
    640 
    641 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(const SDValue &Base, unsigned Offset,
    642                                          unsigned OffsetBits) const {
    643   if ((OffsetBits == 16 && !isUInt<16>(Offset)) ||
    644       (OffsetBits == 8 && !isUInt<8>(Offset)))
    645     return false;
    646 
    647   if (Subtarget->getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS ||
    648       Subtarget->unsafeDSOffsetFoldingEnabled())
    649     return true;
    650 
    651   // On Southern Islands instruction with a negative base value and an offset
    652   // don't seem to work.
    653   return CurDAG->SignBitIsZero(Base);
    654 }
    655 
    656 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
    657                                               SDValue &Offset) const {
    658   SDLoc DL(Addr);
    659   if (CurDAG->isBaseWithConstantOffset(Addr)) {
    660     SDValue N0 = Addr.getOperand(0);
    661     SDValue N1 = Addr.getOperand(1);
    662     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
    663     if (isDSOffsetLegal(N0, C1->getSExtValue(), 16)) {
    664       // (add n0, c0)
    665       Base = N0;
    666       Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
    667       return true;
    668     }
    669   } else if (Addr.getOpcode() == ISD::SUB) {
    670     // sub C, x -> add (sub 0, x), C
    671     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
    672       int64_t ByteOffset = C->getSExtValue();
    673       if (isUInt<16>(ByteOffset)) {
    674         SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
    675 
    676         // XXX - This is kind of hacky. Create a dummy sub node so we can check
    677         // the known bits in isDSOffsetLegal. We need to emit the selected node
    678         // here, so this is thrown away.
    679         SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
    680                                       Zero, Addr.getOperand(1));
    681 
    682         if (isDSOffsetLegal(Sub, ByteOffset, 16)) {
    683           MachineSDNode *MachineSub
    684             = CurDAG->getMachineNode(AMDGPU::V_SUB_I32_e32, DL, MVT::i32,
    685                                      Zero, Addr.getOperand(1));
    686 
    687           Base = SDValue(MachineSub, 0);
    688           Offset = CurDAG->getTargetConstant(ByteOffset, DL, MVT::i16);
    689           return true;
    690         }
    691       }
    692     }
    693   } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
    694     // If we have a constant address, prefer to put the constant into the
    695     // offset. This can save moves to load the constant address since multiple
    696     // operations can share the zero base address register, and enables merging
    697     // into read2 / write2 instructions.
    698 
    699     SDLoc DL(Addr);
    700 
    701     if (isUInt<16>(CAddr->getZExtValue())) {
    702       SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
    703       MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
    704                                  DL, MVT::i32, Zero);
    705       Base = SDValue(MovZero, 0);
    706       Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
    707       return true;
    708     }
    709   }
    710 
    711   // default case
    712   Base = Addr;
    713   Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16);
    714   return true;
    715 }
    716 
    717 // TODO: If offset is too big, put low 16-bit into offset.
    718 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
    719                                                    SDValue &Offset0,
    720                                                    SDValue &Offset1) const {
    721   SDLoc DL(Addr);
    722 
    723   if (CurDAG->isBaseWithConstantOffset(Addr)) {
    724     SDValue N0 = Addr.getOperand(0);
    725     SDValue N1 = Addr.getOperand(1);
    726     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
    727     unsigned DWordOffset0 = C1->getZExtValue() / 4;
    728     unsigned DWordOffset1 = DWordOffset0 + 1;
    729     // (add n0, c0)
    730     if (isDSOffsetLegal(N0, DWordOffset1, 8)) {
    731       Base = N0;
    732       Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8);
    733       Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8);
    734       return true;
    735     }
    736   } else if (Addr.getOpcode() == ISD::SUB) {
    737     // sub C, x -> add (sub 0, x), C
    738     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
    739       unsigned DWordOffset0 = C->getZExtValue() / 4;
    740       unsigned DWordOffset1 = DWordOffset0 + 1;
    741 
    742       if (isUInt<8>(DWordOffset0)) {
    743         SDLoc DL(Addr);
    744         SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
    745 
    746         // XXX - This is kind of hacky. Create a dummy sub node so we can check
    747         // the known bits in isDSOffsetLegal. We need to emit the selected node
    748         // here, so this is thrown away.
    749         SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
    750                                       Zero, Addr.getOperand(1));
    751 
    752         if (isDSOffsetLegal(Sub, DWordOffset1, 8)) {
    753           MachineSDNode *MachineSub
    754             = CurDAG->getMachineNode(AMDGPU::V_SUB_I32_e32, DL, MVT::i32,
    755                                      Zero, Addr.getOperand(1));
    756 
    757           Base = SDValue(MachineSub, 0);
    758           Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8);
    759           Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8);
    760           return true;
    761         }
    762       }
    763     }
    764   } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
    765     unsigned DWordOffset0 = CAddr->getZExtValue() / 4;
    766     unsigned DWordOffset1 = DWordOffset0 + 1;
    767     assert(4 * DWordOffset0 == CAddr->getZExtValue());
    768 
    769     if (isUInt<8>(DWordOffset0) && isUInt<8>(DWordOffset1)) {
    770       SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
    771       MachineSDNode *MovZero
    772         = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
    773                                  DL, MVT::i32, Zero);
    774       Base = SDValue(MovZero, 0);
    775       Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8);
    776       Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8);
    777       return true;
    778     }
    779   }
    780 
    781   // default case
    782   Base = Addr;
    783   Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8);
    784   Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8);
    785   return true;
    786 }
    787 
    788 static bool isLegalMUBUFImmOffset(const ConstantSDNode *Imm) {
    789   return isUInt<12>(Imm->getZExtValue());
    790 }
    791 
    792 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr,
    793                                      SDValue &VAddr, SDValue &SOffset,
    794                                      SDValue &Offset, SDValue &Offen,
    795                                      SDValue &Idxen, SDValue &Addr64,
    796                                      SDValue &GLC, SDValue &SLC,
    797                                      SDValue &TFE) const {
    798   // Subtarget prefers to use flat instruction
    799   if (Subtarget->useFlatForGlobal())
    800     return false;
    801 
    802   SDLoc DL(Addr);
    803 
    804   if (!GLC.getNode())
    805     GLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
    806   if (!SLC.getNode())
    807     SLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
    808   TFE = CurDAG->getTargetConstant(0, DL, MVT::i1);
    809 
    810   Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1);
    811   Offen = CurDAG->getTargetConstant(0, DL, MVT::i1);
    812   Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1);
    813   SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
    814 
    815   if (CurDAG->isBaseWithConstantOffset(Addr)) {
    816     SDValue N0 = Addr.getOperand(0);
    817     SDValue N1 = Addr.getOperand(1);
    818     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
    819 
    820     if (N0.getOpcode() == ISD::ADD) {
    821       // (add (add N2, N3), C1) -> addr64
    822       SDValue N2 = N0.getOperand(0);
    823       SDValue N3 = N0.getOperand(1);
    824       Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
    825       Ptr = N2;
    826       VAddr = N3;
    827     } else {
    828 
    829       // (add N0, C1) -> offset
    830       VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
    831       Ptr = N0;
    832     }
    833 
    834     if (isLegalMUBUFImmOffset(C1)) {
    835       Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
    836       return true;
    837     }
    838 
    839     if (isUInt<32>(C1->getZExtValue())) {
    840       // Illegal offset, store it in soffset.
    841       Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
    842       SOffset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
    843                    CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)),
    844                         0);
    845       return true;
    846     }
    847   }
    848 
    849   if (Addr.getOpcode() == ISD::ADD) {
    850     // (add N0, N1) -> addr64
    851     SDValue N0 = Addr.getOperand(0);
    852     SDValue N1 = Addr.getOperand(1);
    853     Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
    854     Ptr = N0;
    855     VAddr = N1;
    856     Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
    857     return true;
    858   }
    859 
    860   // default case -> offset
    861   VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
    862   Ptr = Addr;
    863   Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
    864 
    865   return true;
    866 }
    867 
    868 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
    869                                            SDValue &VAddr, SDValue &SOffset,
    870                                            SDValue &Offset, SDValue &GLC,
    871                                            SDValue &SLC, SDValue &TFE) const {
    872   SDValue Ptr, Offen, Idxen, Addr64;
    873 
    874   // addr64 bit was removed for volcanic islands.
    875   if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
    876     return false;
    877 
    878   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
    879               GLC, SLC, TFE))
    880     return false;
    881 
    882   ConstantSDNode *C = cast<ConstantSDNode>(Addr64);
    883   if (C->getSExtValue()) {
    884     SDLoc DL(Addr);
    885 
    886     const SITargetLowering& Lowering =
    887       *static_cast<const SITargetLowering*>(getTargetLowering());
    888 
    889     SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0);
    890     return true;
    891   }
    892 
    893   return false;
    894 }
    895 
    896 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
    897                                            SDValue &VAddr, SDValue &SOffset,
    898                                            SDValue &Offset,
    899                                            SDValue &SLC) const {
    900   SLC = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i1);
    901   SDValue GLC, TFE;
    902 
    903   return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE);
    904 }
    905 
    906 bool AMDGPUDAGToDAGISel::SelectMUBUFScratch(SDValue Addr, SDValue &Rsrc,
    907                                             SDValue &VAddr, SDValue &SOffset,
    908                                             SDValue &ImmOffset) const {
    909 
    910   SDLoc DL(Addr);
    911   MachineFunction &MF = CurDAG->getMachineFunction();
    912   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
    913 
    914   Rsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
    915   SOffset = CurDAG->getRegister(Info->getScratchWaveOffsetReg(), MVT::i32);
    916 
    917   // (add n0, c1)
    918   if (CurDAG->isBaseWithConstantOffset(Addr)) {
    919     SDValue N0 = Addr.getOperand(0);
    920     SDValue N1 = Addr.getOperand(1);
    921 
    922     // Offsets in vaddr must be positive.
    923     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
    924     if (isLegalMUBUFImmOffset(C1)) {
    925       VAddr = N0;
    926       ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
    927       return true;
    928     }
    929   }
    930 
    931   // (node)
    932   VAddr = Addr;
    933   ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16);
    934   return true;
    935 }
    936 
    937 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
    938                                            SDValue &SOffset, SDValue &Offset,
    939                                            SDValue &GLC, SDValue &SLC,
    940                                            SDValue &TFE) const {
    941   SDValue Ptr, VAddr, Offen, Idxen, Addr64;
    942   const SIInstrInfo *TII =
    943     static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
    944 
    945   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
    946               GLC, SLC, TFE))
    947     return false;
    948 
    949   if (!cast<ConstantSDNode>(Offen)->getSExtValue() &&
    950       !cast<ConstantSDNode>(Idxen)->getSExtValue() &&
    951       !cast<ConstantSDNode>(Addr64)->getSExtValue()) {
    952     uint64_t Rsrc = TII->getDefaultRsrcDataFormat() |
    953                     APInt::getAllOnesValue(32).getZExtValue(); // Size
    954     SDLoc DL(Addr);
    955 
    956     const SITargetLowering& Lowering =
    957       *static_cast<const SITargetLowering*>(getTargetLowering());
    958 
    959     SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0);
    960     return true;
    961   }
    962   return false;
    963 }
    964 
    965 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
    966                                            SDValue &Soffset, SDValue &Offset
    967                                            ) const {
    968   SDValue GLC, SLC, TFE;
    969 
    970   return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE);
    971 }
    972 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
    973                                            SDValue &Soffset, SDValue &Offset,
    974                                            SDValue &SLC) const {
    975   SDValue GLC, TFE;
    976 
    977   return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE);
    978 }
    979 
    980 bool AMDGPUDAGToDAGISel::SelectMUBUFConstant(SDValue Constant,
    981                                              SDValue &SOffset,
    982                                              SDValue &ImmOffset) const {
    983   SDLoc DL(Constant);
    984   uint32_t Imm = cast<ConstantSDNode>(Constant)->getZExtValue();
    985   uint32_t Overflow = 0;
    986 
    987   if (Imm >= 4096) {
    988     if (Imm <= 4095 + 64) {
    989       // Use an SOffset inline constant for 1..64
    990       Overflow = Imm - 4095;
    991       Imm = 4095;
    992     } else {
    993       // Try to keep the same value in SOffset for adjacent loads, so that
    994       // the corresponding register contents can be re-used.
    995       //
    996       // Load values with all low-bits set into SOffset, so that a larger
    997       // range of values can be covered using s_movk_i32
    998       uint32_t High = (Imm + 1) & ~4095;
    999       uint32_t Low = (Imm + 1) & 4095;
   1000       Imm = Low;
   1001       Overflow = High - 1;
   1002     }
   1003   }
   1004 
   1005   // There is a hardware bug in SI and CI which prevents address clamping in
   1006   // MUBUF instructions from working correctly with SOffsets. The immediate
   1007   // offset is unaffected.
   1008   if (Overflow > 0 &&
   1009       Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)
   1010     return false;
   1011 
   1012   ImmOffset = CurDAG->getTargetConstant(Imm, DL, MVT::i16);
   1013 
   1014   if (Overflow <= 64)
   1015     SOffset = CurDAG->getTargetConstant(Overflow, DL, MVT::i32);
   1016   else
   1017     SOffset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
   1018                       CurDAG->getTargetConstant(Overflow, DL, MVT::i32)),
   1019                       0);
   1020 
   1021   return true;
   1022 }
   1023 
   1024 bool AMDGPUDAGToDAGISel::SelectMUBUFIntrinsicOffset(SDValue Offset,
   1025                                                     SDValue &SOffset,
   1026                                                     SDValue &ImmOffset) const {
   1027   SDLoc DL(Offset);
   1028 
   1029   if (!isa<ConstantSDNode>(Offset))
   1030     return false;
   1031 
   1032   return SelectMUBUFConstant(Offset, SOffset, ImmOffset);
   1033 }
   1034 
   1035 bool AMDGPUDAGToDAGISel::SelectMUBUFIntrinsicVOffset(SDValue Offset,
   1036                                                      SDValue &SOffset,
   1037                                                      SDValue &ImmOffset,
   1038                                                      SDValue &VOffset) const {
   1039   SDLoc DL(Offset);
   1040 
   1041   // Don't generate an unnecessary voffset for constant offsets.
   1042   if (isa<ConstantSDNode>(Offset)) {
   1043     SDValue Tmp1, Tmp2;
   1044 
   1045     // When necessary, use a voffset in <= CI anyway to work around a hardware
   1046     // bug.
   1047     if (Subtarget->getGeneration() > AMDGPUSubtarget::SEA_ISLANDS ||
   1048         SelectMUBUFConstant(Offset, Tmp1, Tmp2))
   1049       return false;
   1050   }
   1051 
   1052   if (CurDAG->isBaseWithConstantOffset(Offset)) {
   1053     SDValue N0 = Offset.getOperand(0);
   1054     SDValue N1 = Offset.getOperand(1);
   1055     if (cast<ConstantSDNode>(N1)->getSExtValue() >= 0 &&
   1056         SelectMUBUFConstant(N1, SOffset, ImmOffset)) {
   1057       VOffset = N0;
   1058       return true;
   1059     }
   1060   }
   1061 
   1062   SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
   1063   ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16);
   1064   VOffset = Offset;
   1065 
   1066   return true;
   1067 }
   1068 
   1069 bool AMDGPUDAGToDAGISel::SelectFlat(SDValue Addr,
   1070                                     SDValue &VAddr,
   1071                                     SDValue &SLC,
   1072                                     SDValue &TFE) const {
   1073   VAddr = Addr;
   1074   TFE = SLC = CurDAG->getTargetConstant(0, SDLoc(), MVT::i1);
   1075   return true;
   1076 }
   1077 
   1078 ///
   1079 /// \param EncodedOffset This is the immediate value that will be encoded
   1080 ///        directly into the instruction.  On SI/CI the \p EncodedOffset
   1081 ///        will be in units of dwords and on VI+ it will be units of bytes.
   1082 static bool isLegalSMRDImmOffset(const AMDGPUSubtarget *ST,
   1083                                  int64_t EncodedOffset) {
   1084   return ST->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS ?
   1085      isUInt<8>(EncodedOffset) : isUInt<20>(EncodedOffset);
   1086 }
   1087 
   1088 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode,
   1089                                           SDValue &Offset, bool &Imm) const {
   1090 
   1091   // FIXME: Handle non-constant offsets.
   1092   ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode);
   1093   if (!C)
   1094     return false;
   1095 
   1096   SDLoc SL(ByteOffsetNode);
   1097   AMDGPUSubtarget::Generation Gen = Subtarget->getGeneration();
   1098   int64_t ByteOffset = C->getSExtValue();
   1099   int64_t EncodedOffset = Gen < AMDGPUSubtarget::VOLCANIC_ISLANDS ?
   1100       ByteOffset >> 2 : ByteOffset;
   1101 
   1102   if (isLegalSMRDImmOffset(Subtarget, EncodedOffset)) {
   1103     Offset = CurDAG->getTargetConstant(EncodedOffset, SL, MVT::i32);
   1104     Imm = true;
   1105     return true;
   1106   }
   1107 
   1108   if (!isUInt<32>(EncodedOffset) || !isUInt<32>(ByteOffset))
   1109     return false;
   1110 
   1111   if (Gen == AMDGPUSubtarget::SEA_ISLANDS && isUInt<32>(EncodedOffset)) {
   1112     // 32-bit Immediates are supported on Sea Islands.
   1113     Offset = CurDAG->getTargetConstant(EncodedOffset, SL, MVT::i32);
   1114   } else {
   1115     SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32);
   1116     Offset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32,
   1117                                             C32Bit), 0);
   1118   }
   1119   Imm = false;
   1120   return true;
   1121 }
   1122 
   1123 bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase,
   1124                                      SDValue &Offset, bool &Imm) const {
   1125 
   1126   SDLoc SL(Addr);
   1127   if (CurDAG->isBaseWithConstantOffset(Addr)) {
   1128     SDValue N0 = Addr.getOperand(0);
   1129     SDValue N1 = Addr.getOperand(1);
   1130 
   1131     if (SelectSMRDOffset(N1, Offset, Imm)) {
   1132       SBase = N0;
   1133       return true;
   1134     }
   1135   }
   1136   SBase = Addr;
   1137   Offset = CurDAG->getTargetConstant(0, SL, MVT::i32);
   1138   Imm = true;
   1139   return true;
   1140 }
   1141 
   1142 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase,
   1143                                        SDValue &Offset) const {
   1144   bool Imm;
   1145   return SelectSMRD(Addr, SBase, Offset, Imm) && Imm;
   1146 }
   1147 
   1148 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase,
   1149                                          SDValue &Offset) const {
   1150 
   1151   if (Subtarget->getGeneration() != AMDGPUSubtarget::SEA_ISLANDS)
   1152     return false;
   1153 
   1154   bool Imm;
   1155   if (!SelectSMRD(Addr, SBase, Offset, Imm))
   1156     return false;
   1157 
   1158   return !Imm && isa<ConstantSDNode>(Offset);
   1159 }
   1160 
   1161 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase,
   1162                                         SDValue &Offset) const {
   1163   bool Imm;
   1164   return SelectSMRD(Addr, SBase, Offset, Imm) && !Imm &&
   1165          !isa<ConstantSDNode>(Offset);
   1166 }
   1167 
   1168 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr,
   1169                                              SDValue &Offset) const {
   1170   bool Imm;
   1171   return SelectSMRDOffset(Addr, Offset, Imm) && Imm;
   1172 }
   1173 
   1174 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr,
   1175                                                SDValue &Offset) const {
   1176   if (Subtarget->getGeneration() != AMDGPUSubtarget::SEA_ISLANDS)
   1177     return false;
   1178 
   1179   bool Imm;
   1180   if (!SelectSMRDOffset(Addr, Offset, Imm))
   1181     return false;
   1182 
   1183   return !Imm && isa<ConstantSDNode>(Offset);
   1184 }
   1185 
   1186 bool AMDGPUDAGToDAGISel::SelectSMRDBufferSgpr(SDValue Addr,
   1187                                               SDValue &Offset) const {
   1188   bool Imm;
   1189   return SelectSMRDOffset(Addr, Offset, Imm) && !Imm &&
   1190          !isa<ConstantSDNode>(Offset);
   1191 }
   1192 
   1193 bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index,
   1194                                             SDValue &Base,
   1195                                             SDValue &Offset) const {
   1196   SDLoc DL(Index);
   1197 
   1198   if (CurDAG->isBaseWithConstantOffset(Index)) {
   1199     SDValue N0 = Index.getOperand(0);
   1200     SDValue N1 = Index.getOperand(1);
   1201     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
   1202 
   1203     // (add n0, c0)
   1204     Base = N0;
   1205     Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
   1206     return true;
   1207   }
   1208 
   1209   if (isa<ConstantSDNode>(Index))
   1210     return false;
   1211 
   1212   Base = Index;
   1213   Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
   1214   return true;
   1215 }
   1216 
   1217 SDNode *AMDGPUDAGToDAGISel::getS_BFE(unsigned Opcode, const SDLoc &DL,
   1218                                      SDValue Val, uint32_t Offset,
   1219                                      uint32_t Width) {
   1220   // Transformation function, pack the offset and width of a BFE into
   1221   // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
   1222   // source, bits [5:0] contain the offset and bits [22:16] the width.
   1223   uint32_t PackedVal = Offset | (Width << 16);
   1224   SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32);
   1225 
   1226   return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst);
   1227 }
   1228 
   1229 void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) {
   1230   // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c)
   1231   // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c)
   1232   // Predicate: 0 < b <= c < 32
   1233 
   1234   const SDValue &Shl = N->getOperand(0);
   1235   ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1));
   1236   ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
   1237 
   1238   if (B && C) {
   1239     uint32_t BVal = B->getZExtValue();
   1240     uint32_t CVal = C->getZExtValue();
   1241 
   1242     if (0 < BVal && BVal <= CVal && CVal < 32) {
   1243       bool Signed = N->getOpcode() == ISD::SRA;
   1244       unsigned Opcode = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
   1245 
   1246       ReplaceNode(N, getS_BFE(Opcode, SDLoc(N), Shl.getOperand(0), CVal - BVal,
   1247                               32 - CVal));
   1248       return;
   1249     }
   1250   }
   1251   SelectCode(N);
   1252 }
   1253 
   1254 void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) {
   1255   switch (N->getOpcode()) {
   1256   case ISD::AND:
   1257     if (N->getOperand(0).getOpcode() == ISD::SRL) {
   1258       // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)"
   1259       // Predicate: isMask(mask)
   1260       const SDValue &Srl = N->getOperand(0);
   1261       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1));
   1262       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
   1263 
   1264       if (Shift && Mask) {
   1265         uint32_t ShiftVal = Shift->getZExtValue();
   1266         uint32_t MaskVal = Mask->getZExtValue();
   1267 
   1268         if (isMask_32(MaskVal)) {
   1269           uint32_t WidthVal = countPopulation(MaskVal);
   1270 
   1271           ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N),
   1272                                   Srl.getOperand(0), ShiftVal, WidthVal));
   1273           return;
   1274         }
   1275       }
   1276     }
   1277     break;
   1278   case ISD::SRL:
   1279     if (N->getOperand(0).getOpcode() == ISD::AND) {
   1280       // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)"
   1281       // Predicate: isMask(mask >> b)
   1282       const SDValue &And = N->getOperand(0);
   1283       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1));
   1284       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1));
   1285 
   1286       if (Shift && Mask) {
   1287         uint32_t ShiftVal = Shift->getZExtValue();
   1288         uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal;
   1289 
   1290         if (isMask_32(MaskVal)) {
   1291           uint32_t WidthVal = countPopulation(MaskVal);
   1292 
   1293           ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N),
   1294                                   And.getOperand(0), ShiftVal, WidthVal));
   1295           return;
   1296         }
   1297       }
   1298     } else if (N->getOperand(0).getOpcode() == ISD::SHL) {
   1299       SelectS_BFEFromShifts(N);
   1300       return;
   1301     }
   1302     break;
   1303   case ISD::SRA:
   1304     if (N->getOperand(0).getOpcode() == ISD::SHL) {
   1305       SelectS_BFEFromShifts(N);
   1306       return;
   1307     }
   1308     break;
   1309 
   1310   case ISD::SIGN_EXTEND_INREG: {
   1311     // sext_inreg (srl x, 16), i8 -> bfe_i32 x, 16, 8
   1312     SDValue Src = N->getOperand(0);
   1313     if (Src.getOpcode() != ISD::SRL)
   1314       break;
   1315 
   1316     const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Src.getOperand(1));
   1317     if (!Amt)
   1318       break;
   1319 
   1320     unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
   1321     ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_I32, SDLoc(N), Src.getOperand(0),
   1322                             Amt->getZExtValue(), Width));
   1323     return;
   1324   }
   1325   }
   1326 
   1327   SelectCode(N);
   1328 }
   1329 
   1330 void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) {
   1331   SDValue Cond = N->getOperand(1);
   1332 
   1333   if (isCBranchSCC(N)) {
   1334     // This brcond will use S_CBRANCH_SCC*, so let tablegen handle it.
   1335     SelectCode(N);
   1336     return;
   1337   }
   1338 
   1339   // The result of VOPC instructions is or'd against ~EXEC before it is
   1340   // written to vcc or another SGPR.  This means that the value '1' is always
   1341   // written to the corresponding bit for results that are masked.  In order
   1342   // to correctly check against vccz, we need to and VCC with the EXEC
   1343   // register in order to clear the value from the masked bits.
   1344 
   1345   SDLoc SL(N);
   1346 
   1347   SDNode *MaskedCond =
   1348         CurDAG->getMachineNode(AMDGPU::S_AND_B64, SL, MVT::i1,
   1349                                CurDAG->getRegister(AMDGPU::EXEC, MVT::i1),
   1350                                Cond);
   1351   SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, AMDGPU::VCC,
   1352                                      SDValue(MaskedCond, 0),
   1353                                      SDValue()); // Passing SDValue() adds a
   1354                                                  // glue output.
   1355   CurDAG->SelectNodeTo(N, AMDGPU::S_CBRANCH_VCCNZ, MVT::Other,
   1356                        N->getOperand(2), // Basic Block
   1357                        VCC.getValue(0),  // Chain
   1358                        VCC.getValue(1)); // Glue
   1359   return;
   1360 }
   1361 
   1362 // This is here because there isn't a way to use the generated sub0_sub1 as the
   1363 // subreg index to EXTRACT_SUBREG in tablegen.
   1364 void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) {
   1365   MemSDNode *Mem = cast<MemSDNode>(N);
   1366   unsigned AS = Mem->getAddressSpace();
   1367   if (AS == AMDGPUAS::FLAT_ADDRESS) {
   1368     SelectCode(N);
   1369     return;
   1370   }
   1371 
   1372   MVT VT = N->getSimpleValueType(0);
   1373   bool Is32 = (VT == MVT::i32);
   1374   SDLoc SL(N);
   1375 
   1376   MachineSDNode *CmpSwap = nullptr;
   1377   if (Subtarget->hasAddr64()) {
   1378     SDValue SRsrc, VAddr, SOffset, Offset, GLC, SLC;
   1379 
   1380     if (SelectMUBUFAddr64(Mem->getBasePtr(), SRsrc, VAddr, SOffset, Offset, SLC)) {
   1381       unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_RTN_ADDR64 :
   1382         AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_RTN_ADDR64;
   1383       SDValue CmpVal = Mem->getOperand(2);
   1384 
   1385       // XXX - Do we care about glue operands?
   1386 
   1387       SDValue Ops[] = {
   1388         CmpVal, VAddr, SRsrc, SOffset, Offset, SLC, Mem->getChain()
   1389       };
   1390 
   1391       CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops);
   1392     }
   1393   }
   1394 
   1395   if (!CmpSwap) {
   1396     SDValue SRsrc, SOffset, Offset, SLC;
   1397     if (SelectMUBUFOffset(Mem->getBasePtr(), SRsrc, SOffset, Offset, SLC)) {
   1398       unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_RTN_OFFSET :
   1399         AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_RTN_OFFSET;
   1400 
   1401       SDValue CmpVal = Mem->getOperand(2);
   1402       SDValue Ops[] = {
   1403         CmpVal, SRsrc, SOffset, Offset, SLC, Mem->getChain()
   1404       };
   1405 
   1406       CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops);
   1407     }
   1408   }
   1409 
   1410   if (!CmpSwap) {
   1411     SelectCode(N);
   1412     return;
   1413   }
   1414 
   1415   MachineSDNode::mmo_iterator MMOs = MF->allocateMemRefsArray(1);
   1416   *MMOs = Mem->getMemOperand();
   1417   CmpSwap->setMemRefs(MMOs, MMOs + 1);
   1418 
   1419   unsigned SubReg = Is32 ? AMDGPU::sub0 : AMDGPU::sub0_sub1;
   1420   SDValue Extract
   1421     = CurDAG->getTargetExtractSubreg(SubReg, SL, VT, SDValue(CmpSwap, 0));
   1422 
   1423   ReplaceUses(SDValue(N, 0), Extract);
   1424   ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 1));
   1425   CurDAG->RemoveDeadNode(N);
   1426 }
   1427 
   1428 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
   1429                                         SDValue &SrcMods) const {
   1430 
   1431   unsigned Mods = 0;
   1432 
   1433   Src = In;
   1434 
   1435   if (Src.getOpcode() == ISD::FNEG) {
   1436     Mods |= SISrcMods::NEG;
   1437     Src = Src.getOperand(0);
   1438   }
   1439 
   1440   if (Src.getOpcode() == ISD::FABS) {
   1441     Mods |= SISrcMods::ABS;
   1442     Src = Src.getOperand(0);
   1443   }
   1444 
   1445   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
   1446 
   1447   return true;
   1448 }
   1449 
   1450 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src,
   1451                                          SDValue &SrcMods) const {
   1452   bool Res = SelectVOP3Mods(In, Src, SrcMods);
   1453   return Res && cast<ConstantSDNode>(SrcMods)->isNullValue();
   1454 }
   1455 
   1456 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src,
   1457                                          SDValue &SrcMods, SDValue &Clamp,
   1458                                          SDValue &Omod) const {
   1459   SDLoc DL(In);
   1460   // FIXME: Handle Clamp and Omod
   1461   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i32);
   1462   Omod = CurDAG->getTargetConstant(0, DL, MVT::i32);
   1463 
   1464   return SelectVOP3Mods(In, Src, SrcMods);
   1465 }
   1466 
   1467 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods0(SDValue In, SDValue &Src,
   1468                                            SDValue &SrcMods, SDValue &Clamp,
   1469                                            SDValue &Omod) const {
   1470   bool Res = SelectVOP3Mods0(In, Src, SrcMods, Clamp, Omod);
   1471 
   1472   return Res && cast<ConstantSDNode>(SrcMods)->isNullValue() &&
   1473                 cast<ConstantSDNode>(Clamp)->isNullValue() &&
   1474                 cast<ConstantSDNode>(Omod)->isNullValue();
   1475 }
   1476 
   1477 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0Clamp(SDValue In, SDValue &Src,
   1478                                               SDValue &SrcMods,
   1479                                               SDValue &Omod) const {
   1480   // FIXME: Handle Omod
   1481   Omod = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
   1482 
   1483   return SelectVOP3Mods(In, Src, SrcMods);
   1484 }
   1485 
   1486 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src,
   1487                                                    SDValue &SrcMods,
   1488                                                    SDValue &Clamp,
   1489                                                    SDValue &Omod) const {
   1490   Clamp = Omod = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
   1491   return SelectVOP3Mods(In, Src, SrcMods);
   1492 }
   1493 
   1494 void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
   1495   MachineFrameInfo *MFI = CurDAG->getMachineFunction().getFrameInfo();
   1496 
   1497   // Handle the perverse case where a frame index is being stored. We don't
   1498   // want to see multiple frame index operands on the same instruction since
   1499   // it complicates things and violates some assumptions about frame index
   1500   // lowering.
   1501   for (int I = MFI->getObjectIndexBegin(), E = MFI->getObjectIndexEnd();
   1502        I != E; ++I) {
   1503     SDValue FI = CurDAG->getTargetFrameIndex(I, MVT::i32);
   1504 
   1505     // It's possible that we have a frame index defined in the function that
   1506     // isn't used in this block.
   1507     if (FI.use_empty())
   1508       continue;
   1509 
   1510     // Skip over the AssertZext inserted during lowering.
   1511     SDValue EffectiveFI = FI;
   1512     auto It = FI->use_begin();
   1513     if (It->getOpcode() == ISD::AssertZext && FI->hasOneUse()) {
   1514       EffectiveFI = SDValue(*It, 0);
   1515       It = EffectiveFI->use_begin();
   1516     }
   1517 
   1518     for (auto It = EffectiveFI->use_begin(); !It.atEnd(); ) {
   1519       SDUse &Use = It.getUse();
   1520       SDNode *User = Use.getUser();
   1521       unsigned OpIdx = It.getOperandNo();
   1522       ++It;
   1523 
   1524       if (MemSDNode *M = dyn_cast<MemSDNode>(User)) {
   1525         unsigned PtrIdx = M->getOpcode() == ISD::STORE ? 2 : 1;
   1526         if (OpIdx == PtrIdx)
   1527           continue;
   1528 
   1529         unsigned OpN = M->getNumOperands();
   1530         SDValue NewOps[8];
   1531 
   1532         assert(OpN < array_lengthof(NewOps));
   1533         for (unsigned Op = 0; Op != OpN; ++Op) {
   1534           if (Op != OpIdx) {
   1535             NewOps[Op] = M->getOperand(Op);
   1536             continue;
   1537           }
   1538 
   1539           MachineSDNode *Mov = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
   1540                                                       SDLoc(M), MVT::i32, FI);
   1541           NewOps[Op] = SDValue(Mov, 0);
   1542         }
   1543 
   1544         CurDAG->UpdateNodeOperands(M, makeArrayRef(NewOps, OpN));
   1545       }
   1546     }
   1547   }
   1548 }
   1549 
   1550 void AMDGPUDAGToDAGISel::PostprocessISelDAG() {
   1551   const AMDGPUTargetLowering& Lowering =
   1552     *static_cast<const AMDGPUTargetLowering*>(getTargetLowering());
   1553   bool IsModified = false;
   1554   do {
   1555     IsModified = false;
   1556     // Go over all selected nodes and try to fold them a bit more
   1557     for (SDNode &Node : CurDAG->allnodes()) {
   1558       MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(&Node);
   1559       if (!MachineNode)
   1560         continue;
   1561 
   1562       SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG);
   1563       if (ResNode != &Node) {
   1564         ReplaceUses(&Node, ResNode);
   1565         IsModified = true;
   1566       }
   1567     }
   1568     CurDAG->RemoveDeadNodes();
   1569   } while (IsModified);
   1570 }
   1571