Home | History | Annotate | Download | only in R600
      1 //===-- SIISelLowering.cpp - SI DAG Lowering Implementation ---------------===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 /// \file
     11 /// \brief Custom DAG lowering for SI
     12 //
     13 //===----------------------------------------------------------------------===//
     14 
     15 #include "SIISelLowering.h"
     16 #include "AMDGPU.h"
     17 #include "AMDGPUIntrinsicInfo.h"
     18 #include "AMDGPUSubtarget.h"
     19 #include "SIInstrInfo.h"
     20 #include "SIMachineFunctionInfo.h"
     21 #include "SIRegisterInfo.h"
     22 #include "llvm/CodeGen/CallingConvLower.h"
     23 #include "llvm/CodeGen/MachineInstrBuilder.h"
     24 #include "llvm/CodeGen/MachineRegisterInfo.h"
     25 #include "llvm/CodeGen/SelectionDAG.h"
     26 #include "llvm/IR/Function.h"
     27 #include "llvm/ADT/SmallString.h"
     28 
     29 using namespace llvm;
     30 
     31 SITargetLowering::SITargetLowering(TargetMachine &TM) :
     32     AMDGPUTargetLowering(TM) {
     33   addRegisterClass(MVT::i1, &AMDGPU::VReg_1RegClass);
     34   addRegisterClass(MVT::i64, &AMDGPU::SReg_64RegClass);
     35 
     36   addRegisterClass(MVT::v32i8, &AMDGPU::SReg_256RegClass);
     37   addRegisterClass(MVT::v64i8, &AMDGPU::SReg_512RegClass);
     38 
     39   addRegisterClass(MVT::i32, &AMDGPU::SReg_32RegClass);
     40   addRegisterClass(MVT::f32, &AMDGPU::VReg_32RegClass);
     41 
     42   addRegisterClass(MVT::f64, &AMDGPU::VReg_64RegClass);
     43   addRegisterClass(MVT::v2i32, &AMDGPU::SReg_64RegClass);
     44   addRegisterClass(MVT::v2f32, &AMDGPU::VReg_64RegClass);
     45 
     46   addRegisterClass(MVT::v4i32, &AMDGPU::SReg_128RegClass);
     47   addRegisterClass(MVT::v4f32, &AMDGPU::VReg_128RegClass);
     48 
     49   addRegisterClass(MVT::v8i32, &AMDGPU::VReg_256RegClass);
     50   addRegisterClass(MVT::v8f32, &AMDGPU::VReg_256RegClass);
     51 
     52   addRegisterClass(MVT::v16i32, &AMDGPU::VReg_512RegClass);
     53   addRegisterClass(MVT::v16f32, &AMDGPU::VReg_512RegClass);
     54 
     55   computeRegisterProperties();
     56 
     57   // Condition Codes
     58   setCondCodeAction(ISD::SETONE, MVT::f32, Expand);
     59   setCondCodeAction(ISD::SETUEQ, MVT::f32, Expand);
     60   setCondCodeAction(ISD::SETUGE, MVT::f32, Expand);
     61   setCondCodeAction(ISD::SETUGT, MVT::f32, Expand);
     62   setCondCodeAction(ISD::SETULE, MVT::f32, Expand);
     63   setCondCodeAction(ISD::SETULT, MVT::f32, Expand);
     64 
     65   setCondCodeAction(ISD::SETONE, MVT::f64, Expand);
     66   setCondCodeAction(ISD::SETUEQ, MVT::f64, Expand);
     67   setCondCodeAction(ISD::SETUGE, MVT::f64, Expand);
     68   setCondCodeAction(ISD::SETUGT, MVT::f64, Expand);
     69   setCondCodeAction(ISD::SETULE, MVT::f64, Expand);
     70   setCondCodeAction(ISD::SETULT, MVT::f64, Expand);
     71 
     72   setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i32, Expand);
     73   setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8f32, Expand);
     74   setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i32, Expand);
     75   setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16f32, Expand);
     76 
     77   setOperationAction(ISD::ADD, MVT::i32, Legal);
     78   setOperationAction(ISD::ADDC, MVT::i32, Legal);
     79   setOperationAction(ISD::ADDE, MVT::i32, Legal);
     80   setOperationAction(ISD::SUBC, MVT::i32, Legal);
     81   setOperationAction(ISD::SUBE, MVT::i32, Legal);
     82 
     83   // We need to custom lower vector stores from local memory
     84   setOperationAction(ISD::LOAD, MVT::v2i32, Custom);
     85   setOperationAction(ISD::LOAD, MVT::v4i32, Custom);
     86   setOperationAction(ISD::LOAD, MVT::v8i32, Custom);
     87   setOperationAction(ISD::LOAD, MVT::v16i32, Custom);
     88 
     89   setOperationAction(ISD::STORE, MVT::v8i32, Custom);
     90   setOperationAction(ISD::STORE, MVT::v16i32, Custom);
     91 
     92   // We need to custom lower loads/stores from private memory
     93   setOperationAction(ISD::LOAD, MVT::i32, Custom);
     94   setOperationAction(ISD::LOAD, MVT::v2i32, Custom);
     95   setOperationAction(ISD::LOAD, MVT::v4i32, Custom);
     96   setOperationAction(ISD::LOAD, MVT::v8i32, Custom);
     97 
     98   setOperationAction(ISD::STORE, MVT::i1, Custom);
     99   setOperationAction(ISD::STORE, MVT::i32, Custom);
    100   setOperationAction(ISD::STORE, MVT::v2i32, Custom);
    101   setOperationAction(ISD::STORE, MVT::v4i32, Custom);
    102 
    103   setOperationAction(ISD::SELECT, MVT::f32, Promote);
    104   AddPromotedToType(ISD::SELECT, MVT::f32, MVT::i32);
    105   setOperationAction(ISD::SELECT, MVT::i64, Custom);
    106   setOperationAction(ISD::SELECT, MVT::f64, Promote);
    107   AddPromotedToType(ISD::SELECT, MVT::f64, MVT::i64);
    108 
    109   setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
    110   setOperationAction(ISD::SELECT_CC, MVT::i32, Expand);
    111   setOperationAction(ISD::SELECT_CC, MVT::i64, Expand);
    112   setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
    113 
    114   setOperationAction(ISD::SETCC, MVT::v2i1, Expand);
    115   setOperationAction(ISD::SETCC, MVT::v4i1, Expand);
    116 
    117   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Legal);
    118   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i1, Custom);
    119   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i1, Custom);
    120 
    121   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Legal);
    122   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Custom);
    123   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8, Custom);
    124 
    125   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Legal);
    126   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Custom);
    127   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Custom);
    128 
    129   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Custom);
    130 
    131   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::Other, Custom);
    132 
    133   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
    134   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::f32, Custom);
    135   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v16i8, Custom);
    136   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v4f32, Custom);
    137 
    138   setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
    139   setOperationAction(ISD::BRCOND, MVT::Other, Custom);
    140 
    141   setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote);
    142   setLoadExtAction(ISD::SEXTLOAD, MVT::i8, Custom);
    143   setLoadExtAction(ISD::SEXTLOAD, MVT::i16, Custom);
    144   setLoadExtAction(ISD::SEXTLOAD, MVT::i32, Expand);
    145   setLoadExtAction(ISD::SEXTLOAD, MVT::v8i16, Expand);
    146   setLoadExtAction(ISD::SEXTLOAD, MVT::v16i16, Expand);
    147 
    148   setLoadExtAction(ISD::ZEXTLOAD, MVT::i1, Promote);
    149   setLoadExtAction(ISD::ZEXTLOAD, MVT::i8, Custom);
    150   setLoadExtAction(ISD::ZEXTLOAD, MVT::i16, Custom);
    151   setLoadExtAction(ISD::ZEXTLOAD, MVT::i32, Expand);
    152 
    153   setLoadExtAction(ISD::EXTLOAD, MVT::i1, Promote);
    154   setLoadExtAction(ISD::EXTLOAD, MVT::i8, Custom);
    155   setLoadExtAction(ISD::EXTLOAD, MVT::i16, Custom);
    156   setLoadExtAction(ISD::EXTLOAD, MVT::i32, Expand);
    157   setLoadExtAction(ISD::EXTLOAD, MVT::f32, Expand);
    158 
    159   setTruncStoreAction(MVT::i32, MVT::i8, Custom);
    160   setTruncStoreAction(MVT::i32, MVT::i16, Custom);
    161   setTruncStoreAction(MVT::f64, MVT::f32, Expand);
    162   setTruncStoreAction(MVT::i64, MVT::i32, Expand);
    163   setTruncStoreAction(MVT::v8i32, MVT::v8i16, Expand);
    164   setTruncStoreAction(MVT::v16i32, MVT::v16i16, Expand);
    165 
    166   setOperationAction(ISD::LOAD, MVT::i1, Custom);
    167 
    168   setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
    169   setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
    170   setOperationAction(ISD::FrameIndex, MVT::i32, Custom);
    171 
    172   // These should use UDIVREM, so set them to expand
    173   setOperationAction(ISD::UDIV, MVT::i64, Expand);
    174   setOperationAction(ISD::UREM, MVT::i64, Expand);
    175 
    176   // We only support LOAD/STORE and vector manipulation ops for vectors
    177   // with > 4 elements.
    178   MVT VecTypes[] = {
    179     MVT::v8i32, MVT::v8f32, MVT::v16i32, MVT::v16f32
    180   };
    181 
    182   for (MVT VT : VecTypes) {
    183     for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
    184       switch(Op) {
    185       case ISD::LOAD:
    186       case ISD::STORE:
    187       case ISD::BUILD_VECTOR:
    188       case ISD::BITCAST:
    189       case ISD::EXTRACT_VECTOR_ELT:
    190       case ISD::INSERT_VECTOR_ELT:
    191       case ISD::CONCAT_VECTORS:
    192       case ISD::INSERT_SUBVECTOR:
    193       case ISD::EXTRACT_SUBVECTOR:
    194         break;
    195       default:
    196         setOperationAction(Op, VT, Expand);
    197         break;
    198       }
    199     }
    200   }
    201 
    202   for (int I = MVT::v1f64; I <= MVT::v8f64; ++I) {
    203     MVT::SimpleValueType VT = static_cast<MVT::SimpleValueType>(I);
    204     setOperationAction(ISD::FTRUNC, VT, Expand);
    205     setOperationAction(ISD::FCEIL, VT, Expand);
    206     setOperationAction(ISD::FFLOOR, VT, Expand);
    207   }
    208 
    209   if (Subtarget->getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS) {
    210     setOperationAction(ISD::FTRUNC, MVT::f64, Legal);
    211     setOperationAction(ISD::FCEIL, MVT::f64, Legal);
    212     setOperationAction(ISD::FFLOOR, MVT::f64, Legal);
    213     setOperationAction(ISD::FRINT, MVT::f64, Legal);
    214   }
    215 
    216   // FIXME: These should be removed and handled the same was as f32 fneg. Source
    217   // modifiers also work for the double instructions.
    218   setOperationAction(ISD::FNEG, MVT::f64, Expand);
    219   setOperationAction(ISD::FABS, MVT::f64, Expand);
    220 
    221   setTargetDAGCombine(ISD::SELECT_CC);
    222   setTargetDAGCombine(ISD::SETCC);
    223 
    224   setTargetDAGCombine(ISD::UINT_TO_FP);
    225 
    226   setSchedulingPreference(Sched::RegPressure);
    227 }
    228 
    229 //===----------------------------------------------------------------------===//
    230 // TargetLowering queries
    231 //===----------------------------------------------------------------------===//
    232 
    233 bool SITargetLowering::allowsUnalignedMemoryAccesses(EVT  VT,
    234                                                      unsigned AddrSpace,
    235                                                      bool *IsFast) const {
    236   if (IsFast)
    237     *IsFast = false;
    238 
    239   // XXX: This depends on the address space and also we may want to revist
    240   // the alignment values we specify in the DataLayout.
    241 
    242   // TODO: I think v3i32 should allow unaligned accesses on CI with DS_READ_B96,
    243   // which isn't a simple VT.
    244   if (!VT.isSimple() || VT == MVT::Other)
    245     return false;
    246 
    247   // XXX - CI changes say "Support for unaligned memory accesses" but I don't
    248   // see what for specifically. The wording everywhere else seems to be the
    249   // same.
    250 
    251   // 3.6.4 - Operations using pairs of VGPRs (for example: double-floats) have
    252   // no alignment restrictions.
    253   if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) {
    254     // Using any pair of GPRs should be the same as any other pair.
    255     if (IsFast)
    256       *IsFast = true;
    257     return VT.bitsGE(MVT::i64);
    258   }
    259 
    260   // XXX - The only mention I see of this in the ISA manual is for LDS direct
    261   // reads the "byte address and must be dword aligned". Is it also true for the
    262   // normal loads and stores?
    263   if (AddrSpace == AMDGPUAS::LOCAL_ADDRESS)
    264     return false;
    265 
    266   // 8.1.6 - For Dword or larger reads or writes, the two LSBs of the
    267   // byte-address are ignored, thus forcing Dword alignment.
    268   if (IsFast)
    269     *IsFast = true;
    270   return VT.bitsGT(MVT::i32);
    271 }
    272 
    273 TargetLoweringBase::LegalizeTypeAction
    274 SITargetLowering::getPreferredVectorAction(EVT VT) const {
    275   if (VT.getVectorNumElements() != 1 && VT.getScalarType().bitsLE(MVT::i16))
    276     return TypeSplitVector;
    277 
    278   return TargetLoweringBase::getPreferredVectorAction(VT);
    279 }
    280 
    281 bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
    282                                                          Type *Ty) const {
    283   const SIInstrInfo *TII =
    284     static_cast<const SIInstrInfo*>(getTargetMachine().getInstrInfo());
    285   return TII->isInlineConstant(Imm);
    286 }
    287 
    288 SDValue SITargetLowering::LowerParameter(SelectionDAG &DAG, EVT VT, EVT MemVT,
    289                                          SDLoc DL, SDValue Chain,
    290                                          unsigned Offset, bool Signed) const {
    291   MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
    292   PointerType *PtrTy = PointerType::get(VT.getTypeForEVT(*DAG.getContext()),
    293                                             AMDGPUAS::CONSTANT_ADDRESS);
    294   SDValue BasePtr =  DAG.getCopyFromReg(Chain, DL,
    295                            MRI.getLiveInVirtReg(AMDGPU::SGPR0_SGPR1), MVT::i64);
    296   SDValue Ptr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr,
    297                                              DAG.getConstant(Offset, MVT::i64));
    298   return DAG.getExtLoad(Signed ? ISD::SEXTLOAD : ISD::ZEXTLOAD, DL, VT, Chain, Ptr,
    299                             MachinePointerInfo(UndefValue::get(PtrTy)), MemVT,
    300                             false, false, MemVT.getSizeInBits() >> 3);
    301 
    302 }
    303 
    304 SDValue SITargetLowering::LowerFormalArguments(
    305                                       SDValue Chain,
    306                                       CallingConv::ID CallConv,
    307                                       bool isVarArg,
    308                                       const SmallVectorImpl<ISD::InputArg> &Ins,
    309                                       SDLoc DL, SelectionDAG &DAG,
    310                                       SmallVectorImpl<SDValue> &InVals) const {
    311 
    312   const TargetRegisterInfo *TRI = getTargetMachine().getRegisterInfo();
    313 
    314   MachineFunction &MF = DAG.getMachineFunction();
    315   FunctionType *FType = MF.getFunction()->getFunctionType();
    316   SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
    317 
    318   assert(CallConv == CallingConv::C);
    319 
    320   SmallVector<ISD::InputArg, 16> Splits;
    321   uint32_t Skipped = 0;
    322 
    323   for (unsigned i = 0, e = Ins.size(), PSInputNum = 0; i != e; ++i) {
    324     const ISD::InputArg &Arg = Ins[i];
    325 
    326     // First check if it's a PS input addr
    327     if (Info->ShaderType == ShaderType::PIXEL && !Arg.Flags.isInReg() &&
    328         !Arg.Flags.isByVal()) {
    329 
    330       assert((PSInputNum <= 15) && "Too many PS inputs!");
    331 
    332       if (!Arg.Used) {
    333         // We can savely skip PS inputs
    334         Skipped |= 1 << i;
    335         ++PSInputNum;
    336         continue;
    337       }
    338 
    339       Info->PSInputAddr |= 1 << PSInputNum++;
    340     }
    341 
    342     // Second split vertices into their elements
    343     if (Info->ShaderType != ShaderType::COMPUTE && Arg.VT.isVector()) {
    344       ISD::InputArg NewArg = Arg;
    345       NewArg.Flags.setSplit();
    346       NewArg.VT = Arg.VT.getVectorElementType();
    347 
    348       // We REALLY want the ORIGINAL number of vertex elements here, e.g. a
    349       // three or five element vertex only needs three or five registers,
    350       // NOT four or eigth.
    351       Type *ParamType = FType->getParamType(Arg.OrigArgIndex);
    352       unsigned NumElements = ParamType->getVectorNumElements();
    353 
    354       for (unsigned j = 0; j != NumElements; ++j) {
    355         Splits.push_back(NewArg);
    356         NewArg.PartOffset += NewArg.VT.getStoreSize();
    357       }
    358 
    359     } else if (Info->ShaderType != ShaderType::COMPUTE) {
    360       Splits.push_back(Arg);
    361     }
    362   }
    363 
    364   SmallVector<CCValAssign, 16> ArgLocs;
    365   CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(),
    366                  getTargetMachine(), ArgLocs, *DAG.getContext());
    367 
    368   // At least one interpolation mode must be enabled or else the GPU will hang.
    369   if (Info->ShaderType == ShaderType::PIXEL && (Info->PSInputAddr & 0x7F) == 0) {
    370     Info->PSInputAddr |= 1;
    371     CCInfo.AllocateReg(AMDGPU::VGPR0);
    372     CCInfo.AllocateReg(AMDGPU::VGPR1);
    373   }
    374 
    375   // The pointer to the list of arguments is stored in SGPR0, SGPR1
    376   if (Info->ShaderType == ShaderType::COMPUTE) {
    377     CCInfo.AllocateReg(AMDGPU::SGPR0);
    378     CCInfo.AllocateReg(AMDGPU::SGPR1);
    379     MF.addLiveIn(AMDGPU::SGPR0_SGPR1, &AMDGPU::SReg_64RegClass);
    380   }
    381 
    382   if (Info->ShaderType == ShaderType::COMPUTE) {
    383     getOriginalFunctionArgs(DAG, DAG.getMachineFunction().getFunction(), Ins,
    384                             Splits);
    385   }
    386 
    387   AnalyzeFormalArguments(CCInfo, Splits);
    388 
    389   for (unsigned i = 0, e = Ins.size(), ArgIdx = 0; i != e; ++i) {
    390 
    391     const ISD::InputArg &Arg = Ins[i];
    392     if (Skipped & (1 << i)) {
    393       InVals.push_back(DAG.getUNDEF(Arg.VT));
    394       continue;
    395     }
    396 
    397     CCValAssign &VA = ArgLocs[ArgIdx++];
    398     EVT VT = VA.getLocVT();
    399 
    400     if (VA.isMemLoc()) {
    401       VT = Ins[i].VT;
    402       EVT MemVT = Splits[i].VT;
    403       // The first 36 bytes of the input buffer contains information about
    404       // thread group and global sizes.
    405       SDValue Arg = LowerParameter(DAG, VT, MemVT,  DL, DAG.getRoot(),
    406                                    36 + VA.getLocMemOffset(),
    407                                    Ins[i].Flags.isSExt());
    408       InVals.push_back(Arg);
    409       continue;
    410     }
    411     assert(VA.isRegLoc() && "Parameter must be in a register!");
    412 
    413     unsigned Reg = VA.getLocReg();
    414 
    415     if (VT == MVT::i64) {
    416       // For now assume it is a pointer
    417       Reg = TRI->getMatchingSuperReg(Reg, AMDGPU::sub0,
    418                                      &AMDGPU::SReg_64RegClass);
    419       Reg = MF.addLiveIn(Reg, &AMDGPU::SReg_64RegClass);
    420       InVals.push_back(DAG.getCopyFromReg(Chain, DL, Reg, VT));
    421       continue;
    422     }
    423 
    424     const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT);
    425 
    426     Reg = MF.addLiveIn(Reg, RC);
    427     SDValue Val = DAG.getCopyFromReg(Chain, DL, Reg, VT);
    428 
    429     if (Arg.VT.isVector()) {
    430 
    431       // Build a vector from the registers
    432       Type *ParamType = FType->getParamType(Arg.OrigArgIndex);
    433       unsigned NumElements = ParamType->getVectorNumElements();
    434 
    435       SmallVector<SDValue, 4> Regs;
    436       Regs.push_back(Val);
    437       for (unsigned j = 1; j != NumElements; ++j) {
    438         Reg = ArgLocs[ArgIdx++].getLocReg();
    439         Reg = MF.addLiveIn(Reg, RC);
    440         Regs.push_back(DAG.getCopyFromReg(Chain, DL, Reg, VT));
    441       }
    442 
    443       // Fill up the missing vector elements
    444       NumElements = Arg.VT.getVectorNumElements() - NumElements;
    445       for (unsigned j = 0; j != NumElements; ++j)
    446         Regs.push_back(DAG.getUNDEF(VT));
    447 
    448       InVals.push_back(DAG.getNode(ISD::BUILD_VECTOR, DL, Arg.VT, Regs));
    449       continue;
    450     }
    451 
    452     InVals.push_back(Val);
    453   }
    454   return Chain;
    455 }
    456 
    457 MachineBasicBlock * SITargetLowering::EmitInstrWithCustomInserter(
    458     MachineInstr * MI, MachineBasicBlock * BB) const {
    459 
    460   MachineBasicBlock::iterator I = *MI;
    461   const SIInstrInfo *TII =
    462     static_cast<const SIInstrInfo*>(getTargetMachine().getInstrInfo());
    463   MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
    464 
    465   switch (MI->getOpcode()) {
    466   default:
    467     return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB);
    468   case AMDGPU::BRANCH: return BB;
    469   case AMDGPU::SI_ADDR64_RSRC: {
    470     unsigned SuperReg = MI->getOperand(0).getReg();
    471     unsigned SubRegLo = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass);
    472     unsigned SubRegHi = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass);
    473     unsigned SubRegHiHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
    474     unsigned SubRegHiLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
    475     BuildMI(*BB, I, MI->getDebugLoc(), TII->get(AMDGPU::S_MOV_B64), SubRegLo)
    476             .addOperand(MI->getOperand(1));
    477     BuildMI(*BB, I, MI->getDebugLoc(), TII->get(AMDGPU::S_MOV_B32), SubRegHiLo)
    478             .addImm(0);
    479     BuildMI(*BB, I, MI->getDebugLoc(), TII->get(AMDGPU::S_MOV_B32), SubRegHiHi)
    480             .addImm(AMDGPU::RSRC_DATA_FORMAT >> 32);
    481     BuildMI(*BB, I, MI->getDebugLoc(), TII->get(AMDGPU::REG_SEQUENCE), SubRegHi)
    482             .addReg(SubRegHiLo)
    483             .addImm(AMDGPU::sub0)
    484             .addReg(SubRegHiHi)
    485             .addImm(AMDGPU::sub1);
    486     BuildMI(*BB, I, MI->getDebugLoc(), TII->get(AMDGPU::REG_SEQUENCE), SuperReg)
    487             .addReg(SubRegLo)
    488             .addImm(AMDGPU::sub0_sub1)
    489             .addReg(SubRegHi)
    490             .addImm(AMDGPU::sub2_sub3);
    491     MI->eraseFromParent();
    492     break;
    493   }
    494   case AMDGPU::V_SUB_F64: {
    495     unsigned DestReg = MI->getOperand(0).getReg();
    496     BuildMI(*BB, I, MI->getDebugLoc(), TII->get(AMDGPU::V_ADD_F64), DestReg)
    497       .addImm(0)  // SRC0 modifiers
    498       .addReg(MI->getOperand(1).getReg())
    499       .addImm(1)  // SRC1 modifiers
    500       .addReg(MI->getOperand(2).getReg())
    501       .addImm(0)  // SRC2 modifiers
    502       .addImm(0)  // src2
    503       .addImm(0)  // CLAMP
    504       .addImm(0); // OMOD
    505     MI->eraseFromParent();
    506     break;
    507   }
    508   case AMDGPU::SI_RegisterStorePseudo: {
    509     MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
    510     unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
    511     MachineInstrBuilder MIB =
    512         BuildMI(*BB, I, MI->getDebugLoc(), TII->get(AMDGPU::SI_RegisterStore),
    513                 Reg);
    514     for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i)
    515       MIB.addOperand(MI->getOperand(i));
    516 
    517     MI->eraseFromParent();
    518     break;
    519   }
    520   case AMDGPU::FABS_SI: {
    521     MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
    522     const SIInstrInfo *TII =
    523       static_cast<const SIInstrInfo*>(getTargetMachine().getInstrInfo());
    524     unsigned Reg = MRI.createVirtualRegister(&AMDGPU::VReg_32RegClass);
    525     BuildMI(*BB, I, MI->getDebugLoc(), TII->get(AMDGPU::V_MOV_B32_e32),
    526             Reg)
    527             .addImm(0x7fffffff);
    528     BuildMI(*BB, I, MI->getDebugLoc(), TII->get(AMDGPU::V_AND_B32_e32),
    529             MI->getOperand(0).getReg())
    530             .addReg(MI->getOperand(1).getReg())
    531             .addReg(Reg);
    532     MI->eraseFromParent();
    533     break;
    534   }
    535   case AMDGPU::FNEG_SI: {
    536     MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
    537     const SIInstrInfo *TII =
    538       static_cast<const SIInstrInfo*>(getTargetMachine().getInstrInfo());
    539     unsigned Reg = MRI.createVirtualRegister(&AMDGPU::VReg_32RegClass);
    540     BuildMI(*BB, I, MI->getDebugLoc(), TII->get(AMDGPU::V_MOV_B32_e32),
    541             Reg)
    542             .addImm(0x80000000);
    543     BuildMI(*BB, I, MI->getDebugLoc(), TII->get(AMDGPU::V_XOR_B32_e32),
    544             MI->getOperand(0).getReg())
    545             .addReg(MI->getOperand(1).getReg())
    546             .addReg(Reg);
    547     MI->eraseFromParent();
    548     break;
    549   }
    550   case AMDGPU::FCLAMP_SI: {
    551     const SIInstrInfo *TII =
    552       static_cast<const SIInstrInfo*>(getTargetMachine().getInstrInfo());
    553     BuildMI(*BB, I, MI->getDebugLoc(), TII->get(AMDGPU::V_ADD_F32_e64),
    554             MI->getOperand(0).getReg())
    555             .addImm(0) // SRC0 modifiers
    556             .addOperand(MI->getOperand(1))
    557             .addImm(0) // SRC1 modifiers
    558             .addImm(0) // SRC1
    559             .addImm(1) // CLAMP
    560             .addImm(0); // OMOD
    561     MI->eraseFromParent();
    562   }
    563   }
    564   return BB;
    565 }
    566 
    567 EVT SITargetLowering::getSetCCResultType(LLVMContext &, EVT VT) const {
    568   if (!VT.isVector()) {
    569     return MVT::i1;
    570   }
    571   return MVT::getVectorVT(MVT::i1, VT.getVectorNumElements());
    572 }
    573 
    574 MVT SITargetLowering::getScalarShiftAmountTy(EVT VT) const {
    575   return MVT::i32;
    576 }
    577 
    578 bool SITargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const {
    579   VT = VT.getScalarType();
    580 
    581   if (!VT.isSimple())
    582     return false;
    583 
    584   switch (VT.getSimpleVT().SimpleTy) {
    585   case MVT::f32:
    586     return false; /* There is V_MAD_F32 for f32 */
    587   case MVT::f64:
    588     return true;
    589   default:
    590     break;
    591   }
    592 
    593   return false;
    594 }
    595 
    596 //===----------------------------------------------------------------------===//
    597 // Custom DAG Lowering Operations
    598 //===----------------------------------------------------------------------===//
    599 
    600 SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
    601   MachineFunction &MF = DAG.getMachineFunction();
    602   SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
    603   switch (Op.getOpcode()) {
    604   default: return AMDGPUTargetLowering::LowerOperation(Op, DAG);
    605   case ISD::BRCOND: return LowerBRCOND(Op, DAG);
    606   case ISD::LOAD: {
    607     LoadSDNode *Load = dyn_cast<LoadSDNode>(Op);
    608     EVT VT = Op.getValueType();
    609 
    610     // These loads are legal.
    611     if (Load->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS &&
    612         VT.isVector() && VT.getVectorNumElements() == 2 &&
    613         VT.getVectorElementType() == MVT::i32)
    614       return SDValue();
    615 
    616     if (Op.getValueType().isVector() &&
    617         (Load->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
    618          Load->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
    619          (Load->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS &&
    620           Op.getValueType().getVectorNumElements() > 4))) {
    621       return SplitVectorLoad(Op, DAG);
    622     } else {
    623       SDValue Result = LowerLOAD(Op, DAG);
    624       assert((!Result.getNode() ||
    625               Result.getNode()->getNumValues() == 2) &&
    626              "Load should return a value and a chain");
    627       return Result;
    628     }
    629   }
    630 
    631   case ISD::SELECT: return LowerSELECT(Op, DAG);
    632   case ISD::STORE: return LowerSTORE(Op, DAG);
    633   case ISD::GlobalAddress: return LowerGlobalAddress(MFI, Op, DAG);
    634   case ISD::INTRINSIC_WO_CHAIN: {
    635     unsigned IntrinsicID =
    636                          cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
    637     EVT VT = Op.getValueType();
    638     SDLoc DL(Op);
    639     //XXX: Hardcoded we only use two to store the pointer to the parameters.
    640     unsigned NumUserSGPRs = 2;
    641     switch (IntrinsicID) {
    642     default: return AMDGPUTargetLowering::LowerOperation(Op, DAG);
    643     case Intrinsic::r600_read_ngroups_x:
    644       return LowerParameter(DAG, VT, VT, DL, DAG.getEntryNode(), 0, false);
    645     case Intrinsic::r600_read_ngroups_y:
    646       return LowerParameter(DAG, VT, VT, DL, DAG.getEntryNode(), 4, false);
    647     case Intrinsic::r600_read_ngroups_z:
    648       return LowerParameter(DAG, VT, VT, DL, DAG.getEntryNode(), 8, false);
    649     case Intrinsic::r600_read_global_size_x:
    650       return LowerParameter(DAG, VT, VT, DL, DAG.getEntryNode(), 12, false);
    651     case Intrinsic::r600_read_global_size_y:
    652       return LowerParameter(DAG, VT, VT, DL, DAG.getEntryNode(), 16, false);
    653     case Intrinsic::r600_read_global_size_z:
    654       return LowerParameter(DAG, VT, VT, DL, DAG.getEntryNode(), 20, false);
    655     case Intrinsic::r600_read_local_size_x:
    656       return LowerParameter(DAG, VT, VT, DL, DAG.getEntryNode(), 24, false);
    657     case Intrinsic::r600_read_local_size_y:
    658       return LowerParameter(DAG, VT, VT, DL, DAG.getEntryNode(), 28, false);
    659     case Intrinsic::r600_read_local_size_z:
    660       return LowerParameter(DAG, VT, VT, DL, DAG.getEntryNode(), 32, false);
    661     case Intrinsic::r600_read_tgid_x:
    662       return CreateLiveInRegister(DAG, &AMDGPU::SReg_32RegClass,
    663                      AMDGPU::SReg_32RegClass.getRegister(NumUserSGPRs + 0), VT);
    664     case Intrinsic::r600_read_tgid_y:
    665       return CreateLiveInRegister(DAG, &AMDGPU::SReg_32RegClass,
    666                      AMDGPU::SReg_32RegClass.getRegister(NumUserSGPRs + 1), VT);
    667     case Intrinsic::r600_read_tgid_z:
    668       return CreateLiveInRegister(DAG, &AMDGPU::SReg_32RegClass,
    669                      AMDGPU::SReg_32RegClass.getRegister(NumUserSGPRs + 2), VT);
    670     case Intrinsic::r600_read_tidig_x:
    671       return CreateLiveInRegister(DAG, &AMDGPU::VReg_32RegClass,
    672                                   AMDGPU::VGPR0, VT);
    673     case Intrinsic::r600_read_tidig_y:
    674       return CreateLiveInRegister(DAG, &AMDGPU::VReg_32RegClass,
    675                                   AMDGPU::VGPR1, VT);
    676     case Intrinsic::r600_read_tidig_z:
    677       return CreateLiveInRegister(DAG, &AMDGPU::VReg_32RegClass,
    678                                   AMDGPU::VGPR2, VT);
    679     case AMDGPUIntrinsic::SI_load_const: {
    680       SDValue Ops [] = {
    681         Op.getOperand(1),
    682         Op.getOperand(2)
    683       };
    684 
    685       MachineMemOperand *MMO = MF.getMachineMemOperand(
    686           MachinePointerInfo(),
    687           MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant,
    688           VT.getSizeInBits() / 8, 4);
    689       return DAG.getMemIntrinsicNode(AMDGPUISD::LOAD_CONSTANT, DL,
    690                                      Op->getVTList(), Ops, VT, MMO);
    691     }
    692     case AMDGPUIntrinsic::SI_sample:
    693       return LowerSampleIntrinsic(AMDGPUISD::SAMPLE, Op, DAG);
    694     case AMDGPUIntrinsic::SI_sampleb:
    695       return LowerSampleIntrinsic(AMDGPUISD::SAMPLEB, Op, DAG);
    696     case AMDGPUIntrinsic::SI_sampled:
    697       return LowerSampleIntrinsic(AMDGPUISD::SAMPLED, Op, DAG);
    698     case AMDGPUIntrinsic::SI_samplel:
    699       return LowerSampleIntrinsic(AMDGPUISD::SAMPLEL, Op, DAG);
    700     case AMDGPUIntrinsic::SI_vs_load_input:
    701       return DAG.getNode(AMDGPUISD::LOAD_INPUT, DL, VT,
    702                          Op.getOperand(1),
    703                          Op.getOperand(2),
    704                          Op.getOperand(3));
    705     }
    706   }
    707 
    708   case ISD::INTRINSIC_VOID:
    709     SDValue Chain = Op.getOperand(0);
    710     unsigned IntrinsicID = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
    711 
    712     switch (IntrinsicID) {
    713       case AMDGPUIntrinsic::SI_tbuffer_store: {
    714         SDLoc DL(Op);
    715         SDValue Ops [] = {
    716           Chain,
    717           Op.getOperand(2),
    718           Op.getOperand(3),
    719           Op.getOperand(4),
    720           Op.getOperand(5),
    721           Op.getOperand(6),
    722           Op.getOperand(7),
    723           Op.getOperand(8),
    724           Op.getOperand(9),
    725           Op.getOperand(10),
    726           Op.getOperand(11),
    727           Op.getOperand(12),
    728           Op.getOperand(13),
    729           Op.getOperand(14)
    730         };
    731         EVT VT = Op.getOperand(3).getValueType();
    732 
    733         MachineMemOperand *MMO = MF.getMachineMemOperand(
    734             MachinePointerInfo(),
    735             MachineMemOperand::MOStore,
    736             VT.getSizeInBits() / 8, 4);
    737         return DAG.getMemIntrinsicNode(AMDGPUISD::TBUFFER_STORE_FORMAT, DL,
    738                                        Op->getVTList(), Ops, VT, MMO);
    739       }
    740       default:
    741         break;
    742     }
    743   }
    744   return SDValue();
    745 }
    746 
    747 /// \brief Helper function for LowerBRCOND
    748 static SDNode *findUser(SDValue Value, unsigned Opcode) {
    749 
    750   SDNode *Parent = Value.getNode();
    751   for (SDNode::use_iterator I = Parent->use_begin(), E = Parent->use_end();
    752        I != E; ++I) {
    753 
    754     if (I.getUse().get() != Value)
    755       continue;
    756 
    757     if (I->getOpcode() == Opcode)
    758       return *I;
    759   }
    760   return nullptr;
    761 }
    762 
    763 /// This transforms the control flow intrinsics to get the branch destination as
    764 /// last parameter, also switches branch target with BR if the need arise
    765 SDValue SITargetLowering::LowerBRCOND(SDValue BRCOND,
    766                                       SelectionDAG &DAG) const {
    767 
    768   SDLoc DL(BRCOND);
    769 
    770   SDNode *Intr = BRCOND.getOperand(1).getNode();
    771   SDValue Target = BRCOND.getOperand(2);
    772   SDNode *BR = nullptr;
    773 
    774   if (Intr->getOpcode() == ISD::SETCC) {
    775     // As long as we negate the condition everything is fine
    776     SDNode *SetCC = Intr;
    777     assert(SetCC->getConstantOperandVal(1) == 1);
    778     assert(cast<CondCodeSDNode>(SetCC->getOperand(2).getNode())->get() ==
    779            ISD::SETNE);
    780     Intr = SetCC->getOperand(0).getNode();
    781 
    782   } else {
    783     // Get the target from BR if we don't negate the condition
    784     BR = findUser(BRCOND, ISD::BR);
    785     Target = BR->getOperand(1);
    786   }
    787 
    788   assert(Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN);
    789 
    790   // Build the result and
    791   SmallVector<EVT, 4> Res;
    792   for (unsigned i = 1, e = Intr->getNumValues(); i != e; ++i)
    793     Res.push_back(Intr->getValueType(i));
    794 
    795   // operands of the new intrinsic call
    796   SmallVector<SDValue, 4> Ops;
    797   Ops.push_back(BRCOND.getOperand(0));
    798   for (unsigned i = 1, e = Intr->getNumOperands(); i != e; ++i)
    799     Ops.push_back(Intr->getOperand(i));
    800   Ops.push_back(Target);
    801 
    802   // build the new intrinsic call
    803   SDNode *Result = DAG.getNode(
    804     Res.size() > 1 ? ISD::INTRINSIC_W_CHAIN : ISD::INTRINSIC_VOID, DL,
    805     DAG.getVTList(Res), Ops).getNode();
    806 
    807   if (BR) {
    808     // Give the branch instruction our target
    809     SDValue Ops[] = {
    810       BR->getOperand(0),
    811       BRCOND.getOperand(2)
    812     };
    813     DAG.MorphNodeTo(BR, ISD::BR, BR->getVTList(), Ops);
    814   }
    815 
    816   SDValue Chain = SDValue(Result, Result->getNumValues() - 1);
    817 
    818   // Copy the intrinsic results to registers
    819   for (unsigned i = 1, e = Intr->getNumValues() - 1; i != e; ++i) {
    820     SDNode *CopyToReg = findUser(SDValue(Intr, i), ISD::CopyToReg);
    821     if (!CopyToReg)
    822       continue;
    823 
    824     Chain = DAG.getCopyToReg(
    825       Chain, DL,
    826       CopyToReg->getOperand(1),
    827       SDValue(Result, i - 1),
    828       SDValue());
    829 
    830     DAG.ReplaceAllUsesWith(SDValue(CopyToReg, 0), CopyToReg->getOperand(0));
    831   }
    832 
    833   // Remove the old intrinsic from the chain
    834   DAG.ReplaceAllUsesOfValueWith(
    835     SDValue(Intr, Intr->getNumValues() - 1),
    836     Intr->getOperand(0));
    837 
    838   return Chain;
    839 }
    840 
    841 SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
    842   SDLoc DL(Op);
    843   LoadSDNode *Load = cast<LoadSDNode>(Op);
    844   SDValue Lowered = AMDGPUTargetLowering::LowerLOAD(Op, DAG);
    845   if (Lowered.getNode())
    846     return Lowered;
    847 
    848   if (Load->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) {
    849     return SDValue();
    850   }
    851 
    852   EVT MemVT = Load->getMemoryVT();
    853 
    854   assert(!MemVT.isVector() && "Private loads should be scalarized");
    855   assert(!MemVT.isFloatingPoint() && "FP loads should be promoted to int");
    856 
    857   SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Load->getBasePtr(),
    858                             DAG.getConstant(2, MVT::i32));
    859 
    860   // FIXME: REGISTER_LOAD should probably have a chain result.
    861   SDValue Chain = Load->getChain();
    862   SDValue LoLoad = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, MVT::i32,
    863                                Chain, Ptr,
    864                                DAG.getTargetConstant(0, MVT::i32),
    865                                Op.getOperand(2));
    866 
    867   SDValue Ret = LoLoad.getValue(0);
    868   if (MemVT.getSizeInBits() == 64) {
    869     // TODO: This needs a test to make sure the right thing is happening with
    870     // the chain. That is hard without general function support.
    871 
    872     SDValue IncPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, Ptr,
    873                                  DAG.getConstant(1, MVT::i32));
    874 
    875     SDValue HiLoad = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, MVT::i32,
    876                                  Chain, IncPtr,
    877                                  DAG.getTargetConstant(0, MVT::i32),
    878                                  Op.getOperand(2));
    879 
    880     Ret = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, LoLoad, HiLoad);
    881     // Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
    882     //                     LoLoad.getValue(1), HiLoad.getValue(1));
    883   }
    884 
    885   SDValue Ops[] = {
    886     Ret,
    887     Chain
    888   };
    889 
    890   return DAG.getMergeValues(Ops, DL);
    891 }
    892 
    893 SDValue SITargetLowering::LowerSampleIntrinsic(unsigned Opcode,
    894                                                const SDValue &Op,
    895                                                SelectionDAG &DAG) const {
    896   return DAG.getNode(Opcode, SDLoc(Op), Op.getValueType(), Op.getOperand(1),
    897                      Op.getOperand(2),
    898                      Op.getOperand(3),
    899                      Op.getOperand(4));
    900 }
    901 
    902 SDValue SITargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
    903   if (Op.getValueType() != MVT::i64)
    904     return SDValue();
    905 
    906   SDLoc DL(Op);
    907   SDValue Cond = Op.getOperand(0);
    908 
    909   SDValue Zero = DAG.getConstant(0, MVT::i32);
    910   SDValue One = DAG.getConstant(1, MVT::i32);
    911 
    912   SDValue LHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(1));
    913   SDValue RHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(2));
    914 
    915   SDValue Lo0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, LHS, Zero);
    916   SDValue Lo1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, RHS, Zero);
    917 
    918   SDValue Lo = DAG.getSelect(DL, MVT::i32, Cond, Lo0, Lo1);
    919 
    920   SDValue Hi0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, LHS, One);
    921   SDValue Hi1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, RHS, One);
    922 
    923   SDValue Hi = DAG.getSelect(DL, MVT::i32, Cond, Hi0, Hi1);
    924 
    925   SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2i32, Lo, Hi);
    926   return DAG.getNode(ISD::BITCAST, DL, MVT::i64, Res);
    927 }
    928 
    929 SDValue SITargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
    930   SDLoc DL(Op);
    931   StoreSDNode *Store = cast<StoreSDNode>(Op);
    932   EVT VT = Store->getMemoryVT();
    933 
    934   // These stores are legal.
    935   if (Store->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS &&
    936       VT.isVector() && VT.getVectorNumElements() == 2 &&
    937       VT.getVectorElementType() == MVT::i32)
    938     return SDValue();
    939 
    940   SDValue Ret = AMDGPUTargetLowering::LowerSTORE(Op, DAG);
    941   if (Ret.getNode())
    942     return Ret;
    943 
    944   if (VT.isVector() && VT.getVectorNumElements() >= 8)
    945       return SplitVectorStore(Op, DAG);
    946 
    947   if (VT == MVT::i1)
    948     return DAG.getTruncStore(Store->getChain(), DL,
    949                         DAG.getSExtOrTrunc(Store->getValue(), DL, MVT::i32),
    950                         Store->getBasePtr(), MVT::i1, Store->getMemOperand());
    951 
    952   if (Store->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS)
    953     return SDValue();
    954 
    955   SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Store->getBasePtr(),
    956                             DAG.getConstant(2, MVT::i32));
    957   SDValue Chain = Store->getChain();
    958   SmallVector<SDValue, 8> Values;
    959 
    960   if (Store->isTruncatingStore()) {
    961     unsigned Mask = 0;
    962     if (Store->getMemoryVT() == MVT::i8) {
    963       Mask = 0xff;
    964     } else if (Store->getMemoryVT() == MVT::i16) {
    965       Mask = 0xffff;
    966     }
    967     SDValue Dst = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, MVT::i32,
    968                               Chain, Store->getBasePtr(),
    969                               DAG.getConstant(0, MVT::i32));
    970     SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, Store->getBasePtr(),
    971                                   DAG.getConstant(0x3, MVT::i32));
    972     SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx,
    973                                    DAG.getConstant(3, MVT::i32));
    974     SDValue MaskedValue = DAG.getNode(ISD::AND, DL, MVT::i32, Store->getValue(),
    975                                       DAG.getConstant(Mask, MVT::i32));
    976     SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, MVT::i32,
    977                                        MaskedValue, ShiftAmt);
    978     SDValue RotrAmt = DAG.getNode(ISD::SUB, DL, MVT::i32,
    979                                   DAG.getConstant(32, MVT::i32), ShiftAmt);
    980     SDValue DstMask = DAG.getNode(ISD::ROTR, DL, MVT::i32,
    981                                   DAG.getConstant(Mask, MVT::i32),
    982                                   RotrAmt);
    983     Dst = DAG.getNode(ISD::AND, DL, MVT::i32, Dst, DstMask);
    984     Dst = DAG.getNode(ISD::OR, DL, MVT::i32, Dst, ShiftedValue);
    985 
    986     Values.push_back(Dst);
    987   } else if (VT == MVT::i64) {
    988     for (unsigned i = 0; i < 2; ++i) {
    989       Values.push_back(DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32,
    990                        Store->getValue(), DAG.getConstant(i, MVT::i32)));
    991     }
    992   } else if (VT == MVT::i128) {
    993     for (unsigned i = 0; i < 2; ++i) {
    994       for (unsigned j = 0; j < 2; ++j) {
    995         Values.push_back(DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32,
    996                            DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64,
    997                            Store->getValue(), DAG.getConstant(i, MVT::i32)),
    998                          DAG.getConstant(j, MVT::i32)));
    999       }
   1000     }
   1001   } else {
   1002     Values.push_back(Store->getValue());
   1003   }
   1004 
   1005   for (unsigned i = 0; i < Values.size(); ++i) {
   1006     SDValue PartPtr = DAG.getNode(ISD::ADD, DL, MVT::i32,
   1007                                   Ptr, DAG.getConstant(i, MVT::i32));
   1008     Chain = DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other,
   1009                         Chain, Values[i], PartPtr,
   1010                         DAG.getTargetConstant(0, MVT::i32));
   1011   }
   1012   return Chain;
   1013 }
   1014 
   1015 //===----------------------------------------------------------------------===//
   1016 // Custom DAG optimizations
   1017 //===----------------------------------------------------------------------===//
   1018 
   1019 SDValue SITargetLowering::performUCharToFloatCombine(SDNode *N,
   1020                                                      DAGCombinerInfo &DCI) {
   1021   EVT VT = N->getValueType(0);
   1022   EVT ScalarVT = VT.getScalarType();
   1023   if (ScalarVT != MVT::f32)
   1024     return SDValue();
   1025 
   1026   SelectionDAG &DAG = DCI.DAG;
   1027   SDLoc DL(N);
   1028 
   1029   SDValue Src = N->getOperand(0);
   1030   EVT SrcVT = Src.getValueType();
   1031 
   1032   // TODO: We could try to match extracting the higher bytes, which would be
   1033   // easier if i8 vectors weren't promoted to i32 vectors, particularly after
   1034   // types are legalized. v4i8 -> v4f32 is probably the only case to worry
   1035   // about in practice.
   1036   if (DCI.isAfterLegalizeVectorOps() && SrcVT == MVT::i32) {
   1037     if (DAG.MaskedValueIsZero(Src, APInt::getHighBitsSet(32, 24))) {
   1038       SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0, DL, VT, Src);
   1039       DCI.AddToWorklist(Cvt.getNode());
   1040       return Cvt;
   1041     }
   1042   }
   1043 
   1044   // We are primarily trying to catch operations on illegal vector types
   1045   // before they are expanded.
   1046   // For scalars, we can use the more flexible method of checking masked bits
   1047   // after legalization.
   1048   if (!DCI.isBeforeLegalize() ||
   1049       !SrcVT.isVector() ||
   1050       SrcVT.getVectorElementType() != MVT::i8) {
   1051     return SDValue();
   1052   }
   1053 
   1054   assert(DCI.isBeforeLegalize() && "Unexpected legal type");
   1055 
   1056   // Weird sized vectors are a pain to handle, but we know 3 is really the same
   1057   // size as 4.
   1058   unsigned NElts = SrcVT.getVectorNumElements();
   1059   if (!SrcVT.isSimple() && NElts != 3)
   1060     return SDValue();
   1061 
   1062   // Handle v4i8 -> v4f32 extload. Replace the v4i8 with a legal i32 load to
   1063   // prevent a mess from expanding to v4i32 and repacking.
   1064   if (ISD::isNormalLoad(Src.getNode()) && Src.hasOneUse()) {
   1065     EVT LoadVT = getEquivalentMemType(*DAG.getContext(), SrcVT);
   1066     EVT RegVT = getEquivalentLoadRegType(*DAG.getContext(), SrcVT);
   1067     EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f32, NElts);
   1068 
   1069     LoadSDNode *Load = cast<LoadSDNode>(Src);
   1070     SDValue NewLoad = DAG.getExtLoad(ISD::ZEXTLOAD, DL, RegVT,
   1071                                      Load->getChain(),
   1072                                      Load->getBasePtr(),
   1073                                      LoadVT,
   1074                                      Load->getMemOperand());
   1075 
   1076     // Make sure successors of the original load stay after it by updating
   1077     // them to use the new Chain.
   1078     DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), NewLoad.getValue(1));
   1079 
   1080     SmallVector<SDValue, 4> Elts;
   1081     if (RegVT.isVector())
   1082       DAG.ExtractVectorElements(NewLoad, Elts);
   1083     else
   1084       Elts.push_back(NewLoad);
   1085 
   1086     SmallVector<SDValue, 4> Ops;
   1087 
   1088     unsigned EltIdx = 0;
   1089     for (SDValue Elt : Elts) {
   1090       unsigned ComponentsInElt = std::min(4u, NElts - 4 * EltIdx);
   1091       for (unsigned I = 0; I < ComponentsInElt; ++I) {
   1092         unsigned Opc = AMDGPUISD::CVT_F32_UBYTE0 + I;
   1093         SDValue Cvt = DAG.getNode(Opc, DL, MVT::f32, Elt);
   1094         DCI.AddToWorklist(Cvt.getNode());
   1095         Ops.push_back(Cvt);
   1096       }
   1097 
   1098       ++EltIdx;
   1099     }
   1100 
   1101     assert(Ops.size() == NElts);
   1102 
   1103     return DAG.getNode(ISD::BUILD_VECTOR, DL, FloatVT, Ops);
   1104   }
   1105 
   1106   return SDValue();
   1107 }
   1108 
   1109 SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
   1110                                             DAGCombinerInfo &DCI) const {
   1111   SelectionDAG &DAG = DCI.DAG;
   1112   SDLoc DL(N);
   1113   EVT VT = N->getValueType(0);
   1114 
   1115   switch (N->getOpcode()) {
   1116     default: return AMDGPUTargetLowering::PerformDAGCombine(N, DCI);
   1117     case ISD::SELECT_CC: {
   1118       ConstantSDNode *True, *False;
   1119       // i1 selectcc(l, r, -1, 0, cc) -> i1 setcc(l, r, cc)
   1120       if ((True = dyn_cast<ConstantSDNode>(N->getOperand(2)))
   1121           && (False = dyn_cast<ConstantSDNode>(N->getOperand(3)))
   1122           && True->isAllOnesValue()
   1123           && False->isNullValue()
   1124           && VT == MVT::i1) {
   1125         return DAG.getNode(ISD::SETCC, DL, VT, N->getOperand(0),
   1126                            N->getOperand(1), N->getOperand(4));
   1127 
   1128       }
   1129       break;
   1130     }
   1131     case ISD::SETCC: {
   1132       SDValue Arg0 = N->getOperand(0);
   1133       SDValue Arg1 = N->getOperand(1);
   1134       SDValue CC = N->getOperand(2);
   1135       ConstantSDNode * C = nullptr;
   1136       ISD::CondCode CCOp = dyn_cast<CondCodeSDNode>(CC)->get();
   1137 
   1138       // i1 setcc (sext(i1), 0, setne) -> i1 setcc(i1, 0, setne)
   1139       if (VT == MVT::i1
   1140           && Arg0.getOpcode() == ISD::SIGN_EXTEND
   1141           && Arg0.getOperand(0).getValueType() == MVT::i1
   1142           && (C = dyn_cast<ConstantSDNode>(Arg1))
   1143           && C->isNullValue()
   1144           && CCOp == ISD::SETNE) {
   1145         return SimplifySetCC(VT, Arg0.getOperand(0),
   1146                              DAG.getConstant(0, MVT::i1), CCOp, true, DCI, DL);
   1147       }
   1148       break;
   1149     }
   1150 
   1151   case AMDGPUISD::CVT_F32_UBYTE0:
   1152   case AMDGPUISD::CVT_F32_UBYTE1:
   1153   case AMDGPUISD::CVT_F32_UBYTE2:
   1154   case AMDGPUISD::CVT_F32_UBYTE3: {
   1155     unsigned Offset = N->getOpcode() - AMDGPUISD::CVT_F32_UBYTE0;
   1156 
   1157     SDValue Src = N->getOperand(0);
   1158     APInt Demanded = APInt::getBitsSet(32, 8 * Offset, 8 * Offset + 8);
   1159 
   1160     APInt KnownZero, KnownOne;
   1161     TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
   1162                                           !DCI.isBeforeLegalizeOps());
   1163     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   1164     if (TLO.ShrinkDemandedConstant(Src, Demanded) ||
   1165         TLI.SimplifyDemandedBits(Src, Demanded, KnownZero, KnownOne, TLO)) {
   1166       DCI.CommitTargetLoweringOpt(TLO);
   1167     }
   1168 
   1169     break;
   1170   }
   1171 
   1172   case ISD::UINT_TO_FP: {
   1173     return performUCharToFloatCombine(N, DCI);
   1174   }
   1175   }
   1176 
   1177   return AMDGPUTargetLowering::PerformDAGCombine(N, DCI);
   1178 }
   1179 
   1180 /// \brief Test if RegClass is one of the VSrc classes
   1181 static bool isVSrc(unsigned RegClass) {
   1182   return AMDGPU::VSrc_32RegClassID == RegClass ||
   1183          AMDGPU::VSrc_64RegClassID == RegClass;
   1184 }
   1185 
   1186 /// \brief Test if RegClass is one of the SSrc classes
   1187 static bool isSSrc(unsigned RegClass) {
   1188   return AMDGPU::SSrc_32RegClassID == RegClass ||
   1189          AMDGPU::SSrc_64RegClassID == RegClass;
   1190 }
   1191 
   1192 /// \brief Analyze the possible immediate value Op
   1193 ///
   1194 /// Returns -1 if it isn't an immediate, 0 if it's and inline immediate
   1195 /// and the immediate value if it's a literal immediate
   1196 int32_t SITargetLowering::analyzeImmediate(const SDNode *N) const {
   1197 
   1198   union {
   1199     int32_t I;
   1200     float F;
   1201   } Imm;
   1202 
   1203   if (const ConstantSDNode *Node = dyn_cast<ConstantSDNode>(N)) {
   1204     if (Node->getZExtValue() >> 32) {
   1205         return -1;
   1206     }
   1207     Imm.I = Node->getSExtValue();
   1208   } else if (const ConstantFPSDNode *Node = dyn_cast<ConstantFPSDNode>(N)) {
   1209     if (N->getValueType(0) != MVT::f32)
   1210       return -1;
   1211     Imm.F = Node->getValueAPF().convertToFloat();
   1212   } else
   1213     return -1; // It isn't an immediate
   1214 
   1215   if ((Imm.I >= -16 && Imm.I <= 64) ||
   1216       Imm.F == 0.5f || Imm.F == -0.5f ||
   1217       Imm.F == 1.0f || Imm.F == -1.0f ||
   1218       Imm.F == 2.0f || Imm.F == -2.0f ||
   1219       Imm.F == 4.0f || Imm.F == -4.0f)
   1220     return 0; // It's an inline immediate
   1221 
   1222   return Imm.I; // It's a literal immediate
   1223 }
   1224 
   1225 /// \brief Try to fold an immediate directly into an instruction
   1226 bool SITargetLowering::foldImm(SDValue &Operand, int32_t &Immediate,
   1227                                bool &ScalarSlotUsed) const {
   1228 
   1229   MachineSDNode *Mov = dyn_cast<MachineSDNode>(Operand);
   1230   const SIInstrInfo *TII =
   1231     static_cast<const SIInstrInfo*>(getTargetMachine().getInstrInfo());
   1232   if (!Mov || !TII->isMov(Mov->getMachineOpcode()))
   1233     return false;
   1234 
   1235   const SDValue &Op = Mov->getOperand(0);
   1236   int32_t Value = analyzeImmediate(Op.getNode());
   1237   if (Value == -1) {
   1238     // Not an immediate at all
   1239     return false;
   1240 
   1241   } else if (Value == 0) {
   1242     // Inline immediates can always be fold
   1243     Operand = Op;
   1244     return true;
   1245 
   1246   } else if (Value == Immediate) {
   1247     // Already fold literal immediate
   1248     Operand = Op;
   1249     return true;
   1250 
   1251   } else if (!ScalarSlotUsed && !Immediate) {
   1252     // Fold this literal immediate
   1253     ScalarSlotUsed = true;
   1254     Immediate = Value;
   1255     Operand = Op;
   1256     return true;
   1257 
   1258   }
   1259 
   1260   return false;
   1261 }
   1262 
   1263 const TargetRegisterClass *SITargetLowering::getRegClassForNode(
   1264                                    SelectionDAG &DAG, const SDValue &Op) const {
   1265   const SIInstrInfo *TII =
   1266     static_cast<const SIInstrInfo*>(getTargetMachine().getInstrInfo());
   1267   const SIRegisterInfo &TRI = TII->getRegisterInfo();
   1268 
   1269   if (!Op->isMachineOpcode()) {
   1270     switch(Op->getOpcode()) {
   1271     case ISD::CopyFromReg: {
   1272       MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
   1273       unsigned Reg = cast<RegisterSDNode>(Op->getOperand(1))->getReg();
   1274       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
   1275         return MRI.getRegClass(Reg);
   1276       }
   1277       return TRI.getPhysRegClass(Reg);
   1278     }
   1279     default:  return nullptr;
   1280     }
   1281   }
   1282   const MCInstrDesc &Desc = TII->get(Op->getMachineOpcode());
   1283   int OpClassID = Desc.OpInfo[Op.getResNo()].RegClass;
   1284   if (OpClassID != -1) {
   1285     return TRI.getRegClass(OpClassID);
   1286   }
   1287   switch(Op.getMachineOpcode()) {
   1288   case AMDGPU::COPY_TO_REGCLASS:
   1289     // Operand 1 is the register class id for COPY_TO_REGCLASS instructions.
   1290     OpClassID = cast<ConstantSDNode>(Op->getOperand(1))->getZExtValue();
   1291 
   1292     // If the COPY_TO_REGCLASS instruction is copying to a VSrc register
   1293     // class, then the register class for the value could be either a
   1294     // VReg or and SReg.  In order to get a more accurate
   1295     if (OpClassID == AMDGPU::VSrc_32RegClassID ||
   1296         OpClassID == AMDGPU::VSrc_64RegClassID) {
   1297       return getRegClassForNode(DAG, Op.getOperand(0));
   1298     }
   1299     return TRI.getRegClass(OpClassID);
   1300   case AMDGPU::EXTRACT_SUBREG: {
   1301     int SubIdx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
   1302     const TargetRegisterClass *SuperClass =
   1303       getRegClassForNode(DAG, Op.getOperand(0));
   1304     return TRI.getSubClassWithSubReg(SuperClass, SubIdx);
   1305   }
   1306   case AMDGPU::REG_SEQUENCE:
   1307     // Operand 0 is the register class id for REG_SEQUENCE instructions.
   1308     return TRI.getRegClass(
   1309       cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue());
   1310   default:
   1311     return getRegClassFor(Op.getSimpleValueType());
   1312   }
   1313 }
   1314 
   1315 /// \brief Does "Op" fit into register class "RegClass" ?
   1316 bool SITargetLowering::fitsRegClass(SelectionDAG &DAG, const SDValue &Op,
   1317                                     unsigned RegClass) const {
   1318   const TargetRegisterInfo *TRI = getTargetMachine().getRegisterInfo();
   1319   const TargetRegisterClass *RC = getRegClassForNode(DAG, Op);
   1320   if (!RC) {
   1321     return false;
   1322   }
   1323   return TRI->getRegClass(RegClass)->hasSubClassEq(RC);
   1324 }
   1325 
   1326 /// \brief Make sure that we don't exeed the number of allowed scalars
   1327 void SITargetLowering::ensureSRegLimit(SelectionDAG &DAG, SDValue &Operand,
   1328                                        unsigned RegClass,
   1329                                        bool &ScalarSlotUsed) const {
   1330 
   1331   // First map the operands register class to a destination class
   1332   if (RegClass == AMDGPU::VSrc_32RegClassID)
   1333     RegClass = AMDGPU::VReg_32RegClassID;
   1334   else if (RegClass == AMDGPU::VSrc_64RegClassID)
   1335     RegClass = AMDGPU::VReg_64RegClassID;
   1336   else
   1337     return;
   1338 
   1339   // Nothing to do if they fit naturally
   1340   if (fitsRegClass(DAG, Operand, RegClass))
   1341     return;
   1342 
   1343   // If the scalar slot isn't used yet use it now
   1344   if (!ScalarSlotUsed) {
   1345     ScalarSlotUsed = true;
   1346     return;
   1347   }
   1348 
   1349   // This is a conservative aproach. It is possible that we can't determine the
   1350   // correct register class and copy too often, but better safe than sorry.
   1351   SDValue RC = DAG.getTargetConstant(RegClass, MVT::i32);
   1352   SDNode *Node = DAG.getMachineNode(TargetOpcode::COPY_TO_REGCLASS, SDLoc(),
   1353                                     Operand.getValueType(), Operand, RC);
   1354   Operand = SDValue(Node, 0);
   1355 }
   1356 
   1357 /// \returns true if \p Node's operands are different from the SDValue list
   1358 /// \p Ops
   1359 static bool isNodeChanged(const SDNode *Node, const std::vector<SDValue> &Ops) {
   1360   for (unsigned i = 0, e = Node->getNumOperands(); i < e; ++i) {
   1361     if (Ops[i].getNode() != Node->getOperand(i).getNode()) {
   1362       return true;
   1363     }
   1364   }
   1365   return false;
   1366 }
   1367 
   1368 /// \brief Try to fold the Nodes operands into the Node
   1369 SDNode *SITargetLowering::foldOperands(MachineSDNode *Node,
   1370                                        SelectionDAG &DAG) const {
   1371 
   1372   // Original encoding (either e32 or e64)
   1373   int Opcode = Node->getMachineOpcode();
   1374   const SIInstrInfo *TII =
   1375     static_cast<const SIInstrInfo*>(getTargetMachine().getInstrInfo());
   1376   const MCInstrDesc *Desc = &TII->get(Opcode);
   1377 
   1378   unsigned NumDefs = Desc->getNumDefs();
   1379   unsigned NumOps = Desc->getNumOperands();
   1380 
   1381   // Commuted opcode if available
   1382   int OpcodeRev = Desc->isCommutable() ? TII->commuteOpcode(Opcode) : -1;
   1383   const MCInstrDesc *DescRev = OpcodeRev == -1 ? nullptr : &TII->get(OpcodeRev);
   1384 
   1385   assert(!DescRev || DescRev->getNumDefs() == NumDefs);
   1386   assert(!DescRev || DescRev->getNumOperands() == NumOps);
   1387 
   1388   // e64 version if available, -1 otherwise
   1389   int OpcodeE64 = AMDGPU::getVOPe64(Opcode);
   1390   const MCInstrDesc *DescE64 = OpcodeE64 == -1 ? nullptr : &TII->get(OpcodeE64);
   1391   int InputModifiers[3] = {0};
   1392 
   1393   assert(!DescE64 || DescE64->getNumDefs() == NumDefs);
   1394 
   1395   int32_t Immediate = Desc->getSize() == 4 ? 0 : -1;
   1396   bool HaveVSrc = false, HaveSSrc = false;
   1397 
   1398   // First figure out what we already have in this instruction.
   1399   for (unsigned i = 0, e = Node->getNumOperands(), Op = NumDefs;
   1400        i != e && Op < NumOps; ++i, ++Op) {
   1401 
   1402     unsigned RegClass = Desc->OpInfo[Op].RegClass;
   1403     if (isVSrc(RegClass))
   1404       HaveVSrc = true;
   1405     else if (isSSrc(RegClass))
   1406       HaveSSrc = true;
   1407     else
   1408       continue;
   1409 
   1410     int32_t Imm = analyzeImmediate(Node->getOperand(i).getNode());
   1411     if (Imm != -1 && Imm != 0) {
   1412       // Literal immediate
   1413       Immediate = Imm;
   1414     }
   1415   }
   1416 
   1417   // If we neither have VSrc nor SSrc, it makes no sense to continue.
   1418   if (!HaveVSrc && !HaveSSrc)
   1419     return Node;
   1420 
   1421   // No scalar allowed when we have both VSrc and SSrc
   1422   bool ScalarSlotUsed = HaveVSrc && HaveSSrc;
   1423 
   1424   // Second go over the operands and try to fold them
   1425   std::vector<SDValue> Ops;
   1426   bool Promote2e64 = false;
   1427   for (unsigned i = 0, e = Node->getNumOperands(), Op = NumDefs;
   1428        i != e && Op < NumOps; ++i, ++Op) {
   1429 
   1430     const SDValue &Operand = Node->getOperand(i);
   1431     Ops.push_back(Operand);
   1432 
   1433     // Already folded immediate?
   1434     if (isa<ConstantSDNode>(Operand.getNode()) ||
   1435         isa<ConstantFPSDNode>(Operand.getNode()))
   1436       continue;
   1437 
   1438     // Is this a VSrc or SSrc operand?
   1439     unsigned RegClass = Desc->OpInfo[Op].RegClass;
   1440     if (isVSrc(RegClass) || isSSrc(RegClass)) {
   1441       // Try to fold the immediates
   1442       if (!foldImm(Ops[i], Immediate, ScalarSlotUsed)) {
   1443         // Folding didn't work, make sure we don't hit the SReg limit.
   1444         ensureSRegLimit(DAG, Ops[i], RegClass, ScalarSlotUsed);
   1445       }
   1446       continue;
   1447     }
   1448 
   1449     if (i == 1 && DescRev && fitsRegClass(DAG, Ops[0], RegClass)) {
   1450 
   1451       unsigned OtherRegClass = Desc->OpInfo[NumDefs].RegClass;
   1452       assert(isVSrc(OtherRegClass) || isSSrc(OtherRegClass));
   1453 
   1454       // Test if it makes sense to swap operands
   1455       if (foldImm(Ops[1], Immediate, ScalarSlotUsed) ||
   1456           (!fitsRegClass(DAG, Ops[1], RegClass) &&
   1457            fitsRegClass(DAG, Ops[1], OtherRegClass))) {
   1458 
   1459         // Swap commutable operands
   1460         std::swap(Ops[0], Ops[1]);
   1461 
   1462         Desc = DescRev;
   1463         DescRev = nullptr;
   1464         continue;
   1465       }
   1466     }
   1467 
   1468     if (Immediate)
   1469       continue;
   1470 
   1471     if (DescE64) {
   1472       // Test if it makes sense to switch to e64 encoding
   1473       unsigned OtherRegClass = DescE64->OpInfo[Op].RegClass;
   1474       if (!isVSrc(OtherRegClass) && !isSSrc(OtherRegClass))
   1475         continue;
   1476 
   1477       int32_t TmpImm = -1;
   1478       if (foldImm(Ops[i], TmpImm, ScalarSlotUsed) ||
   1479           (!fitsRegClass(DAG, Ops[i], RegClass) &&
   1480            fitsRegClass(DAG, Ops[1], OtherRegClass))) {
   1481 
   1482         // Switch to e64 encoding
   1483         Immediate = -1;
   1484         Promote2e64 = true;
   1485         Desc = DescE64;
   1486         DescE64 = nullptr;
   1487       }
   1488     }
   1489 
   1490     if (!DescE64 && !Promote2e64)
   1491       continue;
   1492     if (!Operand.isMachineOpcode())
   1493       continue;
   1494     if (Operand.getMachineOpcode() == AMDGPU::FNEG_SI) {
   1495       Ops.pop_back();
   1496       Ops.push_back(Operand.getOperand(0));
   1497       InputModifiers[i] = 1;
   1498       Promote2e64 = true;
   1499       if (!DescE64)
   1500         continue;
   1501       Desc = DescE64;
   1502       DescE64 = nullptr;
   1503     }
   1504     else if (Operand.getMachineOpcode() == AMDGPU::FABS_SI) {
   1505       Ops.pop_back();
   1506       Ops.push_back(Operand.getOperand(0));
   1507       InputModifiers[i] = 2;
   1508       Promote2e64 = true;
   1509       if (!DescE64)
   1510         continue;
   1511       Desc = DescE64;
   1512       DescE64 = nullptr;
   1513     }
   1514   }
   1515 
   1516   if (Promote2e64) {
   1517     std::vector<SDValue> OldOps(Ops);
   1518     Ops.clear();
   1519     for (unsigned i = 0; i < OldOps.size(); ++i) {
   1520       // src_modifier
   1521       Ops.push_back(DAG.getTargetConstant(InputModifiers[i], MVT::i32));
   1522       Ops.push_back(OldOps[i]);
   1523     }
   1524     // Add the modifier flags while promoting
   1525     for (unsigned i = 0; i < 2; ++i)
   1526       Ops.push_back(DAG.getTargetConstant(0, MVT::i32));
   1527   }
   1528 
   1529   // Add optional chain and glue
   1530   for (unsigned i = NumOps - NumDefs, e = Node->getNumOperands(); i < e; ++i)
   1531     Ops.push_back(Node->getOperand(i));
   1532 
   1533   // Nodes that have a glue result are not CSE'd by getMachineNode(), so in
   1534   // this case a brand new node is always be created, even if the operands
   1535   // are the same as before.  So, manually check if anything has been changed.
   1536   if (Desc->Opcode == Opcode && !isNodeChanged(Node, Ops)) {
   1537     return Node;
   1538   }
   1539 
   1540   // Create a complete new instruction
   1541   return DAG.getMachineNode(Desc->Opcode, SDLoc(Node), Node->getVTList(), Ops);
   1542 }
   1543 
   1544 /// \brief Helper function for adjustWritemask
   1545 static unsigned SubIdx2Lane(unsigned Idx) {
   1546   switch (Idx) {
   1547   default: return 0;
   1548   case AMDGPU::sub0: return 0;
   1549   case AMDGPU::sub1: return 1;
   1550   case AMDGPU::sub2: return 2;
   1551   case AMDGPU::sub3: return 3;
   1552   }
   1553 }
   1554 
   1555 /// \brief Adjust the writemask of MIMG instructions
   1556 void SITargetLowering::adjustWritemask(MachineSDNode *&Node,
   1557                                        SelectionDAG &DAG) const {
   1558   SDNode *Users[4] = { };
   1559   unsigned Lane = 0;
   1560   unsigned OldDmask = Node->getConstantOperandVal(0);
   1561   unsigned NewDmask = 0;
   1562 
   1563   // Try to figure out the used register components
   1564   for (SDNode::use_iterator I = Node->use_begin(), E = Node->use_end();
   1565        I != E; ++I) {
   1566 
   1567     // Abort if we can't understand the usage
   1568     if (!I->isMachineOpcode() ||
   1569         I->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
   1570       return;
   1571 
   1572     // Lane means which subreg of %VGPRa_VGPRb_VGPRc_VGPRd is used.
   1573     // Note that subregs are packed, i.e. Lane==0 is the first bit set
   1574     // in OldDmask, so it can be any of X,Y,Z,W; Lane==1 is the second bit
   1575     // set, etc.
   1576     Lane = SubIdx2Lane(I->getConstantOperandVal(1));
   1577 
   1578     // Set which texture component corresponds to the lane.
   1579     unsigned Comp;
   1580     for (unsigned i = 0, Dmask = OldDmask; i <= Lane; i++) {
   1581       assert(Dmask);
   1582       Comp = countTrailingZeros(Dmask);
   1583       Dmask &= ~(1 << Comp);
   1584     }
   1585 
   1586     // Abort if we have more than one user per component
   1587     if (Users[Lane])
   1588       return;
   1589 
   1590     Users[Lane] = *I;
   1591     NewDmask |= 1 << Comp;
   1592   }
   1593 
   1594   // Abort if there's no change
   1595   if (NewDmask == OldDmask)
   1596     return;
   1597 
   1598   // Adjust the writemask in the node
   1599   std::vector<SDValue> Ops;
   1600   Ops.push_back(DAG.getTargetConstant(NewDmask, MVT::i32));
   1601   for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
   1602     Ops.push_back(Node->getOperand(i));
   1603   Node = (MachineSDNode*)DAG.UpdateNodeOperands(Node, Ops);
   1604 
   1605   // If we only got one lane, replace it with a copy
   1606   // (if NewDmask has only one bit set...)
   1607   if (NewDmask && (NewDmask & (NewDmask-1)) == 0) {
   1608     SDValue RC = DAG.getTargetConstant(AMDGPU::VReg_32RegClassID, MVT::i32);
   1609     SDNode *Copy = DAG.getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
   1610                                       SDLoc(), Users[Lane]->getValueType(0),
   1611                                       SDValue(Node, 0), RC);
   1612     DAG.ReplaceAllUsesWith(Users[Lane], Copy);
   1613     return;
   1614   }
   1615 
   1616   // Update the users of the node with the new indices
   1617   for (unsigned i = 0, Idx = AMDGPU::sub0; i < 4; ++i) {
   1618 
   1619     SDNode *User = Users[i];
   1620     if (!User)
   1621       continue;
   1622 
   1623     SDValue Op = DAG.getTargetConstant(Idx, MVT::i32);
   1624     DAG.UpdateNodeOperands(User, User->getOperand(0), Op);
   1625 
   1626     switch (Idx) {
   1627     default: break;
   1628     case AMDGPU::sub0: Idx = AMDGPU::sub1; break;
   1629     case AMDGPU::sub1: Idx = AMDGPU::sub2; break;
   1630     case AMDGPU::sub2: Idx = AMDGPU::sub3; break;
   1631     }
   1632   }
   1633 }
   1634 
   1635 /// \brief Fold the instructions after selecting them.
   1636 SDNode *SITargetLowering::PostISelFolding(MachineSDNode *Node,
   1637                                           SelectionDAG &DAG) const {
   1638   const SIInstrInfo *TII =
   1639       static_cast<const SIInstrInfo*>(getTargetMachine().getInstrInfo());
   1640   Node = AdjustRegClass(Node, DAG);
   1641 
   1642   if (TII->isMIMG(Node->getMachineOpcode()))
   1643     adjustWritemask(Node, DAG);
   1644 
   1645   return foldOperands(Node, DAG);
   1646 }
   1647 
   1648 /// \brief Assign the register class depending on the number of
   1649 /// bits set in the writemask
   1650 void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr *MI,
   1651                                                      SDNode *Node) const {
   1652   const SIInstrInfo *TII =
   1653       static_cast<const SIInstrInfo*>(getTargetMachine().getInstrInfo());
   1654   if (!TII->isMIMG(MI->getOpcode()))
   1655     return;
   1656 
   1657   unsigned VReg = MI->getOperand(0).getReg();
   1658   unsigned Writemask = MI->getOperand(1).getImm();
   1659   unsigned BitsSet = 0;
   1660   for (unsigned i = 0; i < 4; ++i)
   1661     BitsSet += Writemask & (1 << i) ? 1 : 0;
   1662 
   1663   const TargetRegisterClass *RC;
   1664   switch (BitsSet) {
   1665   default: return;
   1666   case 1:  RC = &AMDGPU::VReg_32RegClass; break;
   1667   case 2:  RC = &AMDGPU::VReg_64RegClass; break;
   1668   case 3:  RC = &AMDGPU::VReg_96RegClass; break;
   1669   }
   1670 
   1671   unsigned NewOpcode = TII->getMaskedMIMGOp(MI->getOpcode(), BitsSet);
   1672   MI->setDesc(TII->get(NewOpcode));
   1673   MachineRegisterInfo &MRI = MI->getParent()->getParent()->getRegInfo();
   1674   MRI.setRegClass(VReg, RC);
   1675 }
   1676 
   1677 MachineSDNode *SITargetLowering::AdjustRegClass(MachineSDNode *N,
   1678                                                 SelectionDAG &DAG) const {
   1679 
   1680   SDLoc DL(N);
   1681   unsigned NewOpcode = N->getMachineOpcode();
   1682 
   1683   switch (N->getMachineOpcode()) {
   1684   default: return N;
   1685   case AMDGPU::S_LOAD_DWORD_IMM:
   1686     NewOpcode = AMDGPU::BUFFER_LOAD_DWORD_ADDR64;
   1687     // Fall-through
   1688   case AMDGPU::S_LOAD_DWORDX2_SGPR:
   1689     if (NewOpcode == N->getMachineOpcode()) {
   1690       NewOpcode = AMDGPU::BUFFER_LOAD_DWORDX2_ADDR64;
   1691     }
   1692     // Fall-through
   1693   case AMDGPU::S_LOAD_DWORDX4_IMM:
   1694   case AMDGPU::S_LOAD_DWORDX4_SGPR: {
   1695     if (NewOpcode == N->getMachineOpcode()) {
   1696       NewOpcode = AMDGPU::BUFFER_LOAD_DWORDX4_ADDR64;
   1697     }
   1698     if (fitsRegClass(DAG, N->getOperand(0), AMDGPU::SReg_64RegClassID)) {
   1699       return N;
   1700     }
   1701     ConstantSDNode *Offset = cast<ConstantSDNode>(N->getOperand(1));
   1702     SDValue Ops[] = {
   1703       SDValue(DAG.getMachineNode(AMDGPU::SI_ADDR64_RSRC, DL, MVT::i128,
   1704                                  DAG.getConstant(0, MVT::i64)), 0),
   1705       N->getOperand(0),
   1706       DAG.getConstant(Offset->getSExtValue() << 2, MVT::i32)
   1707     };
   1708     return DAG.getMachineNode(NewOpcode, DL, N->getVTList(), Ops);
   1709   }
   1710   }
   1711 }
   1712 
   1713 SDValue SITargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
   1714                                                const TargetRegisterClass *RC,
   1715                                                unsigned Reg, EVT VT) const {
   1716   SDValue VReg = AMDGPUTargetLowering::CreateLiveInRegister(DAG, RC, Reg, VT);
   1717 
   1718   return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(DAG.getEntryNode()),
   1719                             cast<RegisterSDNode>(VReg)->getReg(), VT);
   1720 }
   1721