Home | History | Annotate | Download | only in CellSPU 
      1 //===-- SPUISelDAGToDAG.cpp - CellSPU pattern matching inst selector ------===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 // This file defines a pattern matching instruction selector for the Cell SPU,
     11 // converting from a legalized dag to a SPU-target dag.
     12 //
     13 //===----------------------------------------------------------------------===//
     14 
     15 #include "SPU.h"
     16 #include "SPUTargetMachine.h"
     17 #include "SPUHazardRecognizers.h"
     18 #include "SPUFrameLowering.h"
     19 #include "SPUTargetMachine.h"
     20 #include "llvm/CodeGen/MachineConstantPool.h"
     21 #include "llvm/CodeGen/MachineInstrBuilder.h"
     22 #include "llvm/CodeGen/MachineFunction.h"
     23 #include "llvm/CodeGen/SelectionDAG.h"
     24 #include "llvm/CodeGen/SelectionDAGISel.h"
     25 #include "llvm/Target/TargetOptions.h"
     26 #include "llvm/ADT/Statistic.h"
     27 #include "llvm/Constants.h"
     28 #include "llvm/GlobalValue.h"
     29 #include "llvm/Intrinsics.h"
     30 #include "llvm/LLVMContext.h"
     31 #include "llvm/Support/Debug.h"
     32 #include "llvm/Support/ErrorHandling.h"
     33 #include "llvm/Support/MathExtras.h"
     34 #include "llvm/Support/Compiler.h"
     35 #include "llvm/Support/raw_ostream.h"
     36 
     37 using namespace llvm;
     38 
     39 namespace {
     40   //! ConstantSDNode predicate for i32 sign-extended, 10-bit immediates
     41   bool
     42   isI32IntS10Immediate(ConstantSDNode *CN)
     43   {
     44     return isInt<10>(CN->getSExtValue());
     45   }
     46 
     47   //! ConstantSDNode predicate for i32 unsigned 10-bit immediate values
     48   bool
     49   isI32IntU10Immediate(ConstantSDNode *CN)
     50   {
     51     return isUInt<10>(CN->getSExtValue());
     52   }
     53 
     54   //! ConstantSDNode predicate for i16 sign-extended, 10-bit immediate values
     55   bool
     56   isI16IntS10Immediate(ConstantSDNode *CN)
     57   {
     58     return isInt<10>(CN->getSExtValue());
     59   }
     60 
     61   //! ConstantSDNode predicate for i16 unsigned 10-bit immediate values
     62   bool
     63   isI16IntU10Immediate(ConstantSDNode *CN)
     64   {
     65     return isUInt<10>((short) CN->getZExtValue());
     66   }
     67 
     68   //! ConstantSDNode predicate for signed 16-bit values
     69   /*!
     70     \arg CN The constant SelectionDAG node holding the value
     71     \arg Imm The returned 16-bit value, if returning true
     72 
     73     This predicate tests the value in \a CN to see whether it can be
     74     represented as a 16-bit, sign-extended quantity. Returns true if
     75     this is the case.
     76    */
     77   bool
     78   isIntS16Immediate(ConstantSDNode *CN, short &Imm)
     79   {
     80     EVT vt = CN->getValueType(0);
     81     Imm = (short) CN->getZExtValue();
     82     if (vt.getSimpleVT() >= MVT::i1 && vt.getSimpleVT() <= MVT::i16) {
     83       return true;
     84     } else if (vt == MVT::i32) {
     85       int32_t i_val = (int32_t) CN->getZExtValue();
     86       return i_val == SignExtend32<16>(i_val);
     87     } else {
     88       int64_t i_val = (int64_t) CN->getZExtValue();
     89       return i_val == SignExtend64<16>(i_val);
     90     }
     91   }
     92 
     93   //! ConstantFPSDNode predicate for representing floats as 16-bit sign ext.
     94   static bool
     95   isFPS16Immediate(ConstantFPSDNode *FPN, short &Imm)
     96   {
     97     EVT vt = FPN->getValueType(0);
     98     if (vt == MVT::f32) {
     99       int val = FloatToBits(FPN->getValueAPF().convertToFloat());
    100       if (val == SignExtend32<16>(val)) {
    101         Imm = (short) val;
    102         return true;
    103       }
    104     }
    105 
    106     return false;
    107   }
    108 
    109   //! Generate the carry-generate shuffle mask.
    110   SDValue getCarryGenerateShufMask(SelectionDAG &DAG, DebugLoc dl) {
    111     SmallVector<SDValue, 16 > ShufBytes;
    112 
    113     // Create the shuffle mask for "rotating" the borrow up one register slot
    114     // once the borrow is generated.
    115     ShufBytes.push_back(DAG.getConstant(0x04050607, MVT::i32));
    116     ShufBytes.push_back(DAG.getConstant(0x80808080, MVT::i32));
    117     ShufBytes.push_back(DAG.getConstant(0x0c0d0e0f, MVT::i32));
    118     ShufBytes.push_back(DAG.getConstant(0x80808080, MVT::i32));
    119 
    120     return DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
    121                        &ShufBytes[0], ShufBytes.size());
    122   }
    123 
    124   //! Generate the borrow-generate shuffle mask
    125   SDValue getBorrowGenerateShufMask(SelectionDAG &DAG, DebugLoc dl) {
    126     SmallVector<SDValue, 16 > ShufBytes;
    127 
    128     // Create the shuffle mask for "rotating" the borrow up one register slot
    129     // once the borrow is generated.
    130     ShufBytes.push_back(DAG.getConstant(0x04050607, MVT::i32));
    131     ShufBytes.push_back(DAG.getConstant(0xc0c0c0c0, MVT::i32));
    132     ShufBytes.push_back(DAG.getConstant(0x0c0d0e0f, MVT::i32));
    133     ShufBytes.push_back(DAG.getConstant(0xc0c0c0c0, MVT::i32));
    134 
    135     return DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
    136                        &ShufBytes[0], ShufBytes.size());
    137   }
    138 
    139   //===------------------------------------------------------------------===//
    140   /// SPUDAGToDAGISel - Cell SPU-specific code to select SPU machine
    141   /// instructions for SelectionDAG operations.
    142   ///
    143   class SPUDAGToDAGISel :
    144     public SelectionDAGISel
    145   {
    146     const SPUTargetMachine &TM;
    147     const SPUTargetLowering &SPUtli;
    148     unsigned GlobalBaseReg;
    149 
    150   public:
    151     explicit SPUDAGToDAGISel(SPUTargetMachine &tm) :
    152       SelectionDAGISel(tm),
    153       TM(tm),
    154       SPUtli(*tm.getTargetLowering())
    155     { }
    156 
    157     virtual bool runOnMachineFunction(MachineFunction &MF) {
    158       // Make sure we re-emit a set of the global base reg if necessary
    159       GlobalBaseReg = 0;
    160       SelectionDAGISel::runOnMachineFunction(MF);
    161       return true;
    162     }
    163 
    164     /// getI32Imm - Return a target constant with the specified value, of type
    165     /// i32.
    166     inline SDValue getI32Imm(uint32_t Imm) {
    167       return CurDAG->getTargetConstant(Imm, MVT::i32);
    168     }
    169 
    170     /// getSmallIPtrImm - Return a target constant of pointer type.
    171     inline SDValue getSmallIPtrImm(unsigned Imm) {
    172       return CurDAG->getTargetConstant(Imm, SPUtli.getPointerTy());
    173     }
    174 
    175     SDNode *emitBuildVector(SDNode *bvNode) {
    176       EVT vecVT = bvNode->getValueType(0);
    177       DebugLoc dl = bvNode->getDebugLoc();
    178 
    179       // Check to see if this vector can be represented as a CellSPU immediate
    180       // constant by invoking all of the instruction selection predicates:
    181       if (((vecVT == MVT::v8i16) &&
    182            (SPU::get_vec_i16imm(bvNode, *CurDAG, MVT::i16).getNode() != 0)) ||
    183           ((vecVT == MVT::v4i32) &&
    184            ((SPU::get_vec_i16imm(bvNode, *CurDAG, MVT::i32).getNode() != 0) ||
    185             (SPU::get_ILHUvec_imm(bvNode, *CurDAG, MVT::i32).getNode() != 0) ||
    186             (SPU::get_vec_u18imm(bvNode, *CurDAG, MVT::i32).getNode() != 0) ||
    187             (SPU::get_v4i32_imm(bvNode, *CurDAG).getNode() != 0))) ||
    188           ((vecVT == MVT::v2i64) &&
    189            ((SPU::get_vec_i16imm(bvNode, *CurDAG, MVT::i64).getNode() != 0) ||
    190             (SPU::get_ILHUvec_imm(bvNode, *CurDAG, MVT::i64).getNode() != 0) ||
    191             (SPU::get_vec_u18imm(bvNode, *CurDAG, MVT::i64).getNode() != 0)))) {
    192         HandleSDNode Dummy(SDValue(bvNode, 0));
    193         if (SDNode *N = Select(bvNode))
    194           return N;
    195         return Dummy.getValue().getNode();
    196       }
    197 
    198       // No, need to emit a constant pool spill:
    199       std::vector<Constant*> CV;
    200 
    201       for (size_t i = 0; i < bvNode->getNumOperands(); ++i) {
    202         ConstantSDNode *V = cast<ConstantSDNode > (bvNode->getOperand(i));
    203         CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
    204       }
    205 
    206       const Constant *CP = ConstantVector::get(CV);
    207       SDValue CPIdx = CurDAG->getConstantPool(CP, SPUtli.getPointerTy());
    208       unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
    209       SDValue CGPoolOffset =
    210               SPU::LowerConstantPool(CPIdx, *CurDAG, TM);
    211 
    212       HandleSDNode Dummy(CurDAG->getLoad(vecVT, dl,
    213                                          CurDAG->getEntryNode(), CGPoolOffset,
    214                                          MachinePointerInfo::getConstantPool(),
    215                                          false, false, false, Alignment));
    216       CurDAG->ReplaceAllUsesWith(SDValue(bvNode, 0), Dummy.getValue());
    217       if (SDNode *N = SelectCode(Dummy.getValue().getNode()))
    218         return N;
    219       return Dummy.getValue().getNode();
    220     }
    221 
    222     /// Select - Convert the specified operand from a target-independent to a
    223     /// target-specific node if it hasn't already been changed.
    224     SDNode *Select(SDNode *N);
    225 
    226     //! Emit the instruction sequence for i64 shl
    227     SDNode *SelectSHLi64(SDNode *N, EVT OpVT);
    228 
    229     //! Emit the instruction sequence for i64 srl
    230     SDNode *SelectSRLi64(SDNode *N, EVT OpVT);
    231 
    232     //! Emit the instruction sequence for i64 sra
    233     SDNode *SelectSRAi64(SDNode *N, EVT OpVT);
    234 
    235     //! Emit the necessary sequence for loading i64 constants:
    236     SDNode *SelectI64Constant(SDNode *N, EVT OpVT, DebugLoc dl);
    237 
    238     //! Alternate instruction emit sequence for loading i64 constants
    239     SDNode *SelectI64Constant(uint64_t i64const, EVT OpVT, DebugLoc dl);
    240 
    241     //! Returns true if the address N is an A-form (local store) address
    242     bool SelectAFormAddr(SDNode *Op, SDValue N, SDValue &Base,
    243                          SDValue &Index);
    244 
    245     //! D-form address predicate
    246     bool SelectDFormAddr(SDNode *Op, SDValue N, SDValue &Base,
    247                          SDValue &Index);
    248 
    249     /// Alternate D-form address using i7 offset predicate
    250     bool SelectDForm2Addr(SDNode *Op, SDValue N, SDValue &Disp,
    251                           SDValue &Base);
    252 
    253     /// D-form address selection workhorse
    254     bool DFormAddressPredicate(SDNode *Op, SDValue N, SDValue &Disp,
    255                                SDValue &Base, int minOffset, int maxOffset);
    256 
    257     //! Address predicate if N can be expressed as an indexed [r+r] operation.
    258     bool SelectXFormAddr(SDNode *Op, SDValue N, SDValue &Base,
    259                          SDValue &Index);
    260 
    261     /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
    262     /// inline asm expressions.
    263     virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
    264                                               char ConstraintCode,
    265                                               std::vector<SDValue> &OutOps) {
    266       SDValue Op0, Op1;
    267       switch (ConstraintCode) {
    268       default: return true;
    269       case 'm':   // memory
    270         if (!SelectDFormAddr(Op.getNode(), Op, Op0, Op1)
    271             && !SelectAFormAddr(Op.getNode(), Op, Op0, Op1))
    272           SelectXFormAddr(Op.getNode(), Op, Op0, Op1);
    273         break;
    274       case 'o':   // offsetable
    275         if (!SelectDFormAddr(Op.getNode(), Op, Op0, Op1)
    276             && !SelectAFormAddr(Op.getNode(), Op, Op0, Op1)) {
    277           Op0 = Op;
    278           Op1 = getSmallIPtrImm(0);
    279         }
    280         break;
    281       case 'v':   // not offsetable
    282 #if 1
    283         llvm_unreachable("InlineAsmMemoryOperand 'v' constraint not handled.");
    284 #else
    285         SelectAddrIdxOnly(Op, Op, Op0, Op1);
    286         break;
    287 #endif
    288       }
    289 
    290       OutOps.push_back(Op0);
    291       OutOps.push_back(Op1);
    292       return false;
    293     }
    294 
    295     virtual const char *getPassName() const {
    296       return "Cell SPU DAG->DAG Pattern Instruction Selection";
    297     }
    298 
    299   private:
    300     SDValue getRC( MVT );
    301 
    302     // Include the pieces autogenerated from the target description.
    303 #include "SPUGenDAGISel.inc"
    304   };
    305 }
    306 
    307 /*!
    308  \arg Op The ISD instruction operand
    309  \arg N The address to be tested
    310  \arg Base The base address
    311  \arg Index The base address index
    312  */
    313 bool
    314 SPUDAGToDAGISel::SelectAFormAddr(SDNode *Op, SDValue N, SDValue &Base,
    315                     SDValue &Index) {
    316   // These match the addr256k operand type:
    317   EVT OffsVT = MVT::i16;
    318   SDValue Zero = CurDAG->getTargetConstant(0, OffsVT);
    319   int64_t val;
    320 
    321   switch (N.getOpcode()) {
    322   case ISD::Constant:
    323     val = dyn_cast<ConstantSDNode>(N.getNode())->getSExtValue();
    324     Base = CurDAG->getTargetConstant( val , MVT::i32);
    325     Index = Zero;
    326     return true;
    327   case ISD::ConstantPool:
    328   case ISD::GlobalAddress:
    329     report_fatal_error("SPU SelectAFormAddr: Pool/Global not lowered.");
    330     /*NOTREACHED*/
    331 
    332   case ISD::TargetConstant:
    333   case ISD::TargetGlobalAddress:
    334   case ISD::TargetJumpTable:
    335     report_fatal_error("SPUSelectAFormAddr: Target Constant/Pool/Global "
    336                       "not wrapped as A-form address.");
    337     /*NOTREACHED*/
    338 
    339   case SPUISD::AFormAddr:
    340     // Just load from memory if there's only a single use of the location,
    341     // otherwise, this will get handled below with D-form offset addresses
    342     if (N.hasOneUse()) {
    343       SDValue Op0 = N.getOperand(0);
    344       switch (Op0.getOpcode()) {
    345       case ISD::TargetConstantPool:
    346       case ISD::TargetJumpTable:
    347         Base = Op0;
    348         Index = Zero;
    349         return true;
    350 
    351       case ISD::TargetGlobalAddress: {
    352         GlobalAddressSDNode *GSDN = cast<GlobalAddressSDNode>(Op0);
    353         const GlobalValue *GV = GSDN->getGlobal();
    354         if (GV->getAlignment() == 16) {
    355           Base = Op0;
    356           Index = Zero;
    357           return true;
    358         }
    359         break;
    360       }
    361       }
    362     }
    363     break;
    364   }
    365   return false;
    366 }
    367 
    368 bool
    369 SPUDAGToDAGISel::SelectDForm2Addr(SDNode *Op, SDValue N, SDValue &Disp,
    370                                   SDValue &Base) {
    371   const int minDForm2Offset = -(1 << 7);
    372   const int maxDForm2Offset = (1 << 7) - 1;
    373   return DFormAddressPredicate(Op, N, Disp, Base, minDForm2Offset,
    374                                maxDForm2Offset);
    375 }
    376 
    377 /*!
    378   \arg Op The ISD instruction (ignored)
    379   \arg N The address to be tested
    380   \arg Base Base address register/pointer
    381   \arg Index Base address index
    382 
    383   Examine the input address by a base register plus a signed 10-bit
    384   displacement, [r+I10] (D-form address).
    385 
    386   \return true if \a N is a D-form address with \a Base and \a Index set
    387   to non-empty SDValue instances.
    388 */
    389 bool
    390 SPUDAGToDAGISel::SelectDFormAddr(SDNode *Op, SDValue N, SDValue &Base,
    391                                  SDValue &Index) {
    392   return DFormAddressPredicate(Op, N, Base, Index,
    393                                SPUFrameLowering::minFrameOffset(),
    394                                SPUFrameLowering::maxFrameOffset());
    395 }
    396 
    397 bool
    398 SPUDAGToDAGISel::DFormAddressPredicate(SDNode *Op, SDValue N, SDValue &Base,
    399                                       SDValue &Index, int minOffset,
    400                                       int maxOffset) {
    401   unsigned Opc = N.getOpcode();
    402   EVT PtrTy = SPUtli.getPointerTy();
    403 
    404   if (Opc == ISD::FrameIndex) {
    405     // Stack frame index must be less than 512 (divided by 16):
    406     FrameIndexSDNode *FIN = cast<FrameIndexSDNode>(N);
    407     int FI = int(FIN->getIndex());
    408     DEBUG(errs() << "SelectDFormAddr: ISD::FrameIndex = "
    409                << FI << "\n");
    410     if (SPUFrameLowering::FItoStackOffset(FI) < maxOffset) {
    411       Base = CurDAG->getTargetConstant(0, PtrTy);
    412       Index = CurDAG->getTargetFrameIndex(FI, PtrTy);
    413       return true;
    414     }
    415   } else if (Opc == ISD::ADD) {
    416     // Generated by getelementptr
    417     const SDValue Op0 = N.getOperand(0);
    418     const SDValue Op1 = N.getOperand(1);
    419 
    420     if ((Op0.getOpcode() == SPUISD::Hi && Op1.getOpcode() == SPUISD::Lo)
    421         || (Op1.getOpcode() == SPUISD::Hi && Op0.getOpcode() == SPUISD::Lo)) {
    422       Base = CurDAG->getTargetConstant(0, PtrTy);
    423       Index = N;
    424       return true;
    425     } else if (Op1.getOpcode() == ISD::Constant
    426                || Op1.getOpcode() == ISD::TargetConstant) {
    427       ConstantSDNode *CN = cast<ConstantSDNode>(Op1);
    428       int32_t offset = int32_t(CN->getSExtValue());
    429 
    430       if (Op0.getOpcode() == ISD::FrameIndex) {
    431         FrameIndexSDNode *FIN = cast<FrameIndexSDNode>(Op0);
    432         int FI = int(FIN->getIndex());
    433         DEBUG(errs() << "SelectDFormAddr: ISD::ADD offset = " << offset
    434                    << " frame index = " << FI << "\n");
    435 
    436         if (SPUFrameLowering::FItoStackOffset(FI) < maxOffset) {
    437           Base = CurDAG->getTargetConstant(offset, PtrTy);
    438           Index = CurDAG->getTargetFrameIndex(FI, PtrTy);
    439           return true;
    440         }
    441       } else if (offset > minOffset && offset < maxOffset) {
    442         Base = CurDAG->getTargetConstant(offset, PtrTy);
    443         Index = Op0;
    444         return true;
    445       }
    446     } else if (Op0.getOpcode() == ISD::Constant
    447                || Op0.getOpcode() == ISD::TargetConstant) {
    448       ConstantSDNode *CN = cast<ConstantSDNode>(Op0);
    449       int32_t offset = int32_t(CN->getSExtValue());
    450 
    451       if (Op1.getOpcode() == ISD::FrameIndex) {
    452         FrameIndexSDNode *FIN = cast<FrameIndexSDNode>(Op1);
    453         int FI = int(FIN->getIndex());
    454         DEBUG(errs() << "SelectDFormAddr: ISD::ADD offset = " << offset
    455                    << " frame index = " << FI << "\n");
    456 
    457         if (SPUFrameLowering::FItoStackOffset(FI) < maxOffset) {
    458           Base = CurDAG->getTargetConstant(offset, PtrTy);
    459           Index = CurDAG->getTargetFrameIndex(FI, PtrTy);
    460           return true;
    461         }
    462       } else if (offset > minOffset && offset < maxOffset) {
    463         Base = CurDAG->getTargetConstant(offset, PtrTy);
    464         Index = Op1;
    465         return true;
    466       }
    467     }
    468   } else if (Opc == SPUISD::IndirectAddr) {
    469     // Indirect with constant offset -> D-Form address
    470     const SDValue Op0 = N.getOperand(0);
    471     const SDValue Op1 = N.getOperand(1);
    472 
    473     if (Op0.getOpcode() == SPUISD::Hi
    474         && Op1.getOpcode() == SPUISD::Lo) {
    475       // (SPUindirect (SPUhi <arg>, 0), (SPUlo <arg>, 0))
    476       Base = CurDAG->getTargetConstant(0, PtrTy);
    477       Index = N;
    478       return true;
    479     } else if (isa<ConstantSDNode>(Op0) || isa<ConstantSDNode>(Op1)) {
    480       int32_t offset = 0;
    481       SDValue idxOp;
    482 
    483       if (isa<ConstantSDNode>(Op1)) {
    484         ConstantSDNode *CN = cast<ConstantSDNode>(Op1);
    485         offset = int32_t(CN->getSExtValue());
    486         idxOp = Op0;
    487       } else if (isa<ConstantSDNode>(Op0)) {
    488         ConstantSDNode *CN = cast<ConstantSDNode>(Op0);
    489         offset = int32_t(CN->getSExtValue());
    490         idxOp = Op1;
    491       }
    492 
    493       if (offset >= minOffset && offset <= maxOffset) {
    494         Base = CurDAG->getTargetConstant(offset, PtrTy);
    495         Index = idxOp;
    496         return true;
    497       }
    498     }
    499   } else if (Opc == SPUISD::AFormAddr) {
    500     Base = CurDAG->getTargetConstant(0, N.getValueType());
    501     Index = N;
    502     return true;
    503   } else if (Opc == SPUISD::LDRESULT) {
    504     Base = CurDAG->getTargetConstant(0, N.getValueType());
    505     Index = N;
    506     return true;
    507   } else if (Opc == ISD::Register
    508            ||Opc == ISD::CopyFromReg
    509            ||Opc == ISD::UNDEF
    510            ||Opc == ISD::Constant) {
    511     unsigned OpOpc = Op->getOpcode();
    512 
    513     if (OpOpc == ISD::STORE || OpOpc == ISD::LOAD) {
    514       // Direct load/store without getelementptr
    515       SDValue Offs;
    516 
    517       Offs = ((OpOpc == ISD::STORE) ? Op->getOperand(3) : Op->getOperand(2));
    518 
    519       if (Offs.getOpcode() == ISD::Constant || Offs.getOpcode() == ISD::UNDEF) {
    520         if (Offs.getOpcode() == ISD::UNDEF)
    521           Offs = CurDAG->getTargetConstant(0, Offs.getValueType());
    522 
    523         Base = Offs;
    524         Index = N;
    525         return true;
    526       }
    527     } else {
    528       /* If otherwise unadorned, default to D-form address with 0 offset: */
    529       if (Opc == ISD::CopyFromReg) {
    530         Index = N.getOperand(1);
    531       } else {
    532         Index = N;
    533       }
    534 
    535       Base = CurDAG->getTargetConstant(0, Index.getValueType());
    536       return true;
    537     }
    538   }
    539 
    540   return false;
    541 }
    542 
    543 /*!
    544   \arg Op The ISD instruction operand
    545   \arg N The address operand
    546   \arg Base The base pointer operand
    547   \arg Index The offset/index operand
    548 
    549   If the address \a N can be expressed as an A-form or D-form address, returns
    550   false.  Otherwise, creates two operands, Base and Index that will become the
    551   (r)(r) X-form address.
    552 */
    553 bool
    554 SPUDAGToDAGISel::SelectXFormAddr(SDNode *Op, SDValue N, SDValue &Base,
    555                                  SDValue &Index) {
    556   if (!SelectAFormAddr(Op, N, Base, Index)
    557       && !SelectDFormAddr(Op, N, Base, Index)) {
    558     // If the address is neither A-form or D-form, punt and use an X-form
    559     // address:
    560     Base = N.getOperand(1);
    561     Index = N.getOperand(0);
    562     return true;
    563   }
    564 
    565   return false;
    566 }
    567 
    568 /*!
    569  Utility function to use with COPY_TO_REGCLASS instructions. Returns a SDValue
    570  to be used as the last parameter of a
    571 CurDAG->getMachineNode(COPY_TO_REGCLASS,..., ) function call
    572  \arg VT the value type for which we want a register class
    573 */
    574 SDValue SPUDAGToDAGISel::getRC( MVT VT ) {
    575   switch( VT.SimpleTy ) {
    576   case MVT::i8:
    577     return CurDAG->getTargetConstant(SPU::R8CRegClass.getID(), MVT::i32);
    578   case MVT::i16:
    579     return CurDAG->getTargetConstant(SPU::R16CRegClass.getID(), MVT::i32);
    580   case MVT::i32:
    581     return CurDAG->getTargetConstant(SPU::R32CRegClass.getID(), MVT::i32);
    582   case MVT::f32:
    583     return CurDAG->getTargetConstant(SPU::R32FPRegClass.getID(), MVT::i32);
    584   case MVT::i64:
    585     return CurDAG->getTargetConstant(SPU::R64CRegClass.getID(), MVT::i32);
    586   case MVT::i128:
    587     return CurDAG->getTargetConstant(SPU::GPRCRegClass.getID(), MVT::i32);
    588   case MVT::v16i8:
    589   case MVT::v8i16:
    590   case MVT::v4i32:
    591   case MVT::v4f32:
    592   case MVT::v2i64:
    593   case MVT::v2f64:
    594     return CurDAG->getTargetConstant(SPU::VECREGRegClass.getID(), MVT::i32);
    595   default:
    596     assert( false && "add a new case here" );
    597     return SDValue();
    598   }
    599 }
    600 
    601 //! Convert the operand from a target-independent to a target-specific node
    602 /*!
    603  */
    604 SDNode *
    605 SPUDAGToDAGISel::Select(SDNode *N) {
    606   unsigned Opc = N->getOpcode();
    607   int n_ops = -1;
    608   unsigned NewOpc = 0;
    609   EVT OpVT = N->getValueType(0);
    610   SDValue Ops[8];
    611   DebugLoc dl = N->getDebugLoc();
    612 
    613   if (N->isMachineOpcode())
    614     return NULL;   // Already selected.
    615 
    616   if (Opc == ISD::FrameIndex) {
    617     int FI = cast<FrameIndexSDNode>(N)->getIndex();
    618     SDValue TFI = CurDAG->getTargetFrameIndex(FI, N->getValueType(0));
    619     SDValue Imm0 = CurDAG->getTargetConstant(0, N->getValueType(0));
    620 
    621     if (FI < 128) {
    622       NewOpc = SPU::AIr32;
    623       Ops[0] = TFI;
    624       Ops[1] = Imm0;
    625       n_ops = 2;
    626     } else {
    627       NewOpc = SPU::Ar32;
    628       Ops[0] = CurDAG->getRegister(SPU::R1, N->getValueType(0));
    629       Ops[1] = SDValue(CurDAG->getMachineNode(SPU::ILAr32, dl,
    630                                               N->getValueType(0), TFI),
    631                        0);
    632       n_ops = 2;
    633     }
    634   } else if (Opc == ISD::Constant && OpVT == MVT::i64) {
    635     // Catch the i64 constants that end up here. Note: The backend doesn't
    636     // attempt to legalize the constant (it's useless because DAGCombiner
    637     // will insert 64-bit constants and we can't stop it).
    638     return SelectI64Constant(N, OpVT, N->getDebugLoc());
    639   } else if ((Opc == ISD::ZERO_EXTEND || Opc == ISD::ANY_EXTEND)
    640              && OpVT == MVT::i64) {
    641     SDValue Op0 = N->getOperand(0);
    642     EVT Op0VT = Op0.getValueType();
    643     EVT Op0VecVT = EVT::getVectorVT(*CurDAG->getContext(),
    644                                     Op0VT, (128 / Op0VT.getSizeInBits()));
    645     EVT OpVecVT = EVT::getVectorVT(*CurDAG->getContext(),
    646                                    OpVT, (128 / OpVT.getSizeInBits()));
    647     SDValue shufMask;
    648 
    649     switch (Op0VT.getSimpleVT().SimpleTy) {
    650     default:
    651       report_fatal_error("CellSPU Select: Unhandled zero/any extend EVT");
    652       /*NOTREACHED*/
    653     case MVT::i32:
    654       shufMask = CurDAG->getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
    655                                  CurDAG->getConstant(0x80808080, MVT::i32),
    656                                  CurDAG->getConstant(0x00010203, MVT::i32),
    657                                  CurDAG->getConstant(0x80808080, MVT::i32),
    658                                  CurDAG->getConstant(0x08090a0b, MVT::i32));
    659       break;
    660 
    661     case MVT::i16:
    662       shufMask = CurDAG->getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
    663                                  CurDAG->getConstant(0x80808080, MVT::i32),
    664                                  CurDAG->getConstant(0x80800203, MVT::i32),
    665                                  CurDAG->getConstant(0x80808080, MVT::i32),
    666                                  CurDAG->getConstant(0x80800a0b, MVT::i32));
    667       break;
    668 
    669     case MVT::i8:
    670       shufMask = CurDAG->getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
    671                                  CurDAG->getConstant(0x80808080, MVT::i32),
    672                                  CurDAG->getConstant(0x80808003, MVT::i32),
    673                                  CurDAG->getConstant(0x80808080, MVT::i32),
    674                                  CurDAG->getConstant(0x8080800b, MVT::i32));
    675       break;
    676     }
    677 
    678     SDNode *shufMaskLoad = emitBuildVector(shufMask.getNode());
    679 
    680     HandleSDNode PromoteScalar(CurDAG->getNode(SPUISD::PREFSLOT2VEC, dl,
    681                                                Op0VecVT, Op0));
    682 
    683     SDValue PromScalar;
    684     if (SDNode *N = SelectCode(PromoteScalar.getValue().getNode()))
    685       PromScalar = SDValue(N, 0);
    686     else
    687       PromScalar = PromoteScalar.getValue();
    688 
    689     SDValue zextShuffle =
    690             CurDAG->getNode(SPUISD::SHUFB, dl, OpVecVT,
    691                             PromScalar, PromScalar,
    692                             SDValue(shufMaskLoad, 0));
    693 
    694     HandleSDNode Dummy2(zextShuffle);
    695     if (SDNode *N = SelectCode(Dummy2.getValue().getNode()))
    696       zextShuffle = SDValue(N, 0);
    697     else
    698       zextShuffle = Dummy2.getValue();
    699     HandleSDNode Dummy(CurDAG->getNode(SPUISD::VEC2PREFSLOT, dl, OpVT,
    700                                        zextShuffle));
    701 
    702     CurDAG->ReplaceAllUsesWith(N, Dummy.getValue().getNode());
    703     SelectCode(Dummy.getValue().getNode());
    704     return Dummy.getValue().getNode();
    705   } else if (Opc == ISD::ADD && (OpVT == MVT::i64 || OpVT == MVT::v2i64)) {
    706     SDNode *CGLoad =
    707             emitBuildVector(getCarryGenerateShufMask(*CurDAG, dl).getNode());
    708 
    709     HandleSDNode Dummy(CurDAG->getNode(SPUISD::ADD64_MARKER, dl, OpVT,
    710                                        N->getOperand(0), N->getOperand(1),
    711                                        SDValue(CGLoad, 0)));
    712 
    713     CurDAG->ReplaceAllUsesWith(N, Dummy.getValue().getNode());
    714     if (SDNode *N = SelectCode(Dummy.getValue().getNode()))
    715       return N;
    716     return Dummy.getValue().getNode();
    717   } else if (Opc == ISD::SUB && (OpVT == MVT::i64 || OpVT == MVT::v2i64)) {
    718     SDNode *CGLoad =
    719             emitBuildVector(getBorrowGenerateShufMask(*CurDAG, dl).getNode());
    720 
    721     HandleSDNode Dummy(CurDAG->getNode(SPUISD::SUB64_MARKER, dl, OpVT,
    722                                        N->getOperand(0), N->getOperand(1),
    723                                        SDValue(CGLoad, 0)));
    724 
    725     CurDAG->ReplaceAllUsesWith(N, Dummy.getValue().getNode());
    726     if (SDNode *N = SelectCode(Dummy.getValue().getNode()))
    727       return N;
    728     return Dummy.getValue().getNode();
    729   } else if (Opc == ISD::MUL && (OpVT == MVT::i64 || OpVT == MVT::v2i64)) {
    730     SDNode *CGLoad =
    731             emitBuildVector(getCarryGenerateShufMask(*CurDAG, dl).getNode());
    732 
    733     HandleSDNode Dummy(CurDAG->getNode(SPUISD::MUL64_MARKER, dl, OpVT,
    734                                        N->getOperand(0), N->getOperand(1),
    735                                        SDValue(CGLoad, 0)));
    736     CurDAG->ReplaceAllUsesWith(N, Dummy.getValue().getNode());
    737     if (SDNode *N = SelectCode(Dummy.getValue().getNode()))
    738       return N;
    739     return Dummy.getValue().getNode();
    740   } else if (Opc == ISD::TRUNCATE) {
    741     SDValue Op0 = N->getOperand(0);
    742     if ((Op0.getOpcode() == ISD::SRA || Op0.getOpcode() == ISD::SRL)
    743         && OpVT == MVT::i32
    744         && Op0.getValueType() == MVT::i64) {
    745       // Catch (truncate:i32 ([sra|srl]:i64 arg, c), where c >= 32
    746       //
    747       // Take advantage of the fact that the upper 32 bits are in the
    748       // i32 preferred slot and avoid shuffle gymnastics:
    749       ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op0.getOperand(1));
    750       if (CN != 0) {
    751         unsigned shift_amt = unsigned(CN->getZExtValue());
    752 
    753         if (shift_amt >= 32) {
    754           SDNode *hi32 =
    755                   CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, dl, OpVT,
    756                                          Op0.getOperand(0), getRC(MVT::i32));
    757 
    758           shift_amt -= 32;
    759           if (shift_amt > 0) {
    760             // Take care of the additional shift, if present:
    761             SDValue shift = CurDAG->getTargetConstant(shift_amt, MVT::i32);
    762             unsigned Opc = SPU::ROTMAIr32_i32;
    763 
    764             if (Op0.getOpcode() == ISD::SRL)
    765               Opc = SPU::ROTMr32;
    766 
    767             hi32 = CurDAG->getMachineNode(Opc, dl, OpVT, SDValue(hi32, 0),
    768                                           shift);
    769           }
    770 
    771           return hi32;
    772         }
    773       }
    774     }
    775   } else if (Opc == ISD::SHL) {
    776     if (OpVT == MVT::i64)
    777       return SelectSHLi64(N, OpVT);
    778   } else if (Opc == ISD::SRL) {
    779     if (OpVT == MVT::i64)
    780       return SelectSRLi64(N, OpVT);
    781   } else if (Opc == ISD::SRA) {
    782     if (OpVT == MVT::i64)
    783       return SelectSRAi64(N, OpVT);
    784   } else if (Opc == ISD::FNEG
    785              && (OpVT == MVT::f64 || OpVT == MVT::v2f64)) {
    786     DebugLoc dl = N->getDebugLoc();
    787     // Check if the pattern is a special form of DFNMS:
    788     // (fneg (fsub (fmul R64FP:$rA, R64FP:$rB), R64FP:$rC))
    789     SDValue Op0 = N->getOperand(0);
    790     if (Op0.getOpcode() == ISD::FSUB) {
    791       SDValue Op00 = Op0.getOperand(0);
    792       if (Op00.getOpcode() == ISD::FMUL) {
    793         unsigned Opc = SPU::DFNMSf64;
    794         if (OpVT == MVT::v2f64)
    795           Opc = SPU::DFNMSv2f64;
    796 
    797         return CurDAG->getMachineNode(Opc, dl, OpVT,
    798                                       Op00.getOperand(0),
    799                                       Op00.getOperand(1),
    800                                       Op0.getOperand(1));
    801       }
    802     }
    803 
    804     SDValue negConst = CurDAG->getConstant(0x8000000000000000ULL, MVT::i64);
    805     SDNode *signMask = 0;
    806     unsigned Opc = SPU::XORfneg64;
    807 
    808     if (OpVT == MVT::f64) {
    809       signMask = SelectI64Constant(negConst.getNode(), MVT::i64, dl);
    810     } else if (OpVT == MVT::v2f64) {
    811       Opc = SPU::XORfnegvec;
    812       signMask = emitBuildVector(CurDAG->getNode(ISD::BUILD_VECTOR, dl,
    813                                                  MVT::v2i64,
    814                                                  negConst, negConst).getNode());
    815     }
    816 
    817     return CurDAG->getMachineNode(Opc, dl, OpVT,
    818                                   N->getOperand(0), SDValue(signMask, 0));
    819   } else if (Opc == ISD::FABS) {
    820     if (OpVT == MVT::f64) {
    821       SDNode *signMask = SelectI64Constant(0x7fffffffffffffffULL, MVT::i64, dl);
    822       return CurDAG->getMachineNode(SPU::ANDfabs64, dl, OpVT,
    823                                     N->getOperand(0), SDValue(signMask, 0));
    824     } else if (OpVT == MVT::v2f64) {
    825       SDValue absConst = CurDAG->getConstant(0x7fffffffffffffffULL, MVT::i64);
    826       SDValue absVec = CurDAG->getNode(ISD::BUILD_VECTOR, dl, MVT::v2i64,
    827                                        absConst, absConst);
    828       SDNode *signMask = emitBuildVector(absVec.getNode());
    829       return CurDAG->getMachineNode(SPU::ANDfabsvec, dl, OpVT,
    830                                     N->getOperand(0), SDValue(signMask, 0));
    831     }
    832   } else if (Opc == SPUISD::LDRESULT) {
    833     // Custom select instructions for LDRESULT
    834     EVT VT = N->getValueType(0);
    835     SDValue Arg = N->getOperand(0);
    836     SDValue Chain = N->getOperand(1);
    837     SDNode *Result;
    838 
    839     Result = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, dl, VT,
    840                                     MVT::Other, Arg,
    841                                     getRC( VT.getSimpleVT()), Chain);
    842     return Result;
    843 
    844   } else if (Opc == SPUISD::IndirectAddr) {
    845     // Look at the operands: SelectCode() will catch the cases that aren't
    846     // specifically handled here.
    847     //
    848     // SPUInstrInfo catches the following patterns:
    849     // (SPUindirect (SPUhi ...), (SPUlo ...))
    850     // (SPUindirect $sp, imm)
    851     EVT VT = N->getValueType(0);
    852     SDValue Op0 = N->getOperand(0);
    853     SDValue Op1 = N->getOperand(1);
    854     RegisterSDNode *RN;
    855 
    856     if ((Op0.getOpcode() != SPUISD::Hi && Op1.getOpcode() != SPUISD::Lo)
    857         || (Op0.getOpcode() == ISD::Register
    858             && ((RN = dyn_cast<RegisterSDNode>(Op0.getNode())) != 0
    859                 && RN->getReg() != SPU::R1))) {
    860       NewOpc = SPU::Ar32;
    861       Ops[1] = Op1;
    862       if (Op1.getOpcode() == ISD::Constant) {
    863         ConstantSDNode *CN = cast<ConstantSDNode>(Op1);
    864         Op1 = CurDAG->getTargetConstant(CN->getSExtValue(), VT);
    865         if (isInt<10>(CN->getSExtValue())) {
    866           NewOpc = SPU::AIr32;
    867           Ops[1] = Op1;
    868         } else {
    869           Ops[1] = SDValue(CurDAG->getMachineNode(SPU::ILr32, dl,
    870                                                   N->getValueType(0),
    871                                                   Op1),
    872                            0);
    873         }
    874       }
    875       Ops[0] = Op0;
    876       n_ops = 2;
    877     }
    878   }
    879 
    880   if (n_ops > 0) {
    881     if (N->hasOneUse())
    882       return CurDAG->SelectNodeTo(N, NewOpc, OpVT, Ops, n_ops);
    883     else
    884       return CurDAG->getMachineNode(NewOpc, dl, OpVT, Ops, n_ops);
    885   } else
    886     return SelectCode(N);
    887 }
    888 
    889 /*!
    890  * Emit the instruction sequence for i64 left shifts. The basic algorithm
    891  * is to fill the bottom two word slots with zeros so that zeros are shifted
    892  * in as the entire quadword is shifted left.
    893  *
    894  * \note This code could also be used to implement v2i64 shl.
    895  *
    896  * @param Op The shl operand
    897  * @param OpVT Op's machine value value type (doesn't need to be passed, but
    898  * makes life easier.)
    899  * @return The SDNode with the entire instruction sequence
    900  */
    901 SDNode *
    902 SPUDAGToDAGISel::SelectSHLi64(SDNode *N, EVT OpVT) {
    903   SDValue Op0 = N->getOperand(0);
    904   EVT VecVT = EVT::getVectorVT(*CurDAG->getContext(),
    905                                OpVT, (128 / OpVT.getSizeInBits()));
    906   SDValue ShiftAmt = N->getOperand(1);
    907   EVT ShiftAmtVT = ShiftAmt.getValueType();
    908   SDNode *VecOp0, *SelMask, *ZeroFill, *Shift = 0;
    909   SDValue SelMaskVal;
    910   DebugLoc dl = N->getDebugLoc();
    911 
    912   VecOp0 = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, dl, VecVT,
    913                                   Op0, getRC(MVT::v2i64) );
    914   SelMaskVal = CurDAG->getTargetConstant(0xff00ULL, MVT::i16);
    915   SelMask = CurDAG->getMachineNode(SPU::FSMBIv2i64, dl, VecVT, SelMaskVal);
    916   ZeroFill = CurDAG->getMachineNode(SPU::ILv2i64, dl, VecVT,
    917                                     CurDAG->getTargetConstant(0, OpVT));
    918   VecOp0 = CurDAG->getMachineNode(SPU::SELBv2i64, dl, VecVT,
    919                                   SDValue(ZeroFill, 0),
    920                                   SDValue(VecOp0, 0),
    921                                   SDValue(SelMask, 0));
    922 
    923   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(ShiftAmt)) {
    924     unsigned bytes = unsigned(CN->getZExtValue()) >> 3;
    925     unsigned bits = unsigned(CN->getZExtValue()) & 7;
    926 
    927     if (bytes > 0) {
    928       Shift =
    929         CurDAG->getMachineNode(SPU::SHLQBYIv2i64, dl, VecVT,
    930                                SDValue(VecOp0, 0),
    931                                CurDAG->getTargetConstant(bytes, ShiftAmtVT));
    932     }
    933 
    934     if (bits > 0) {
    935       Shift =
    936         CurDAG->getMachineNode(SPU::SHLQBIIv2i64, dl, VecVT,
    937                                SDValue((Shift != 0 ? Shift : VecOp0), 0),
    938                                CurDAG->getTargetConstant(bits, ShiftAmtVT));
    939     }
    940   } else {
    941     SDNode *Bytes =
    942       CurDAG->getMachineNode(SPU::ROTMIr32, dl, ShiftAmtVT,
    943                              ShiftAmt,
    944                              CurDAG->getTargetConstant(3, ShiftAmtVT));
    945     SDNode *Bits =
    946       CurDAG->getMachineNode(SPU::ANDIr32, dl, ShiftAmtVT,
    947                              ShiftAmt,
    948                              CurDAG->getTargetConstant(7, ShiftAmtVT));
    949     Shift =
    950       CurDAG->getMachineNode(SPU::SHLQBYv2i64, dl, VecVT,
    951                              SDValue(VecOp0, 0), SDValue(Bytes, 0));
    952     Shift =
    953       CurDAG->getMachineNode(SPU::SHLQBIv2i64, dl, VecVT,
    954                              SDValue(Shift, 0), SDValue(Bits, 0));
    955   }
    956 
    957   return CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, dl,
    958                                 OpVT, SDValue(Shift, 0), getRC(MVT::i64));
    959 }
    960 
    961 /*!
    962  * Emit the instruction sequence for i64 logical right shifts.
    963  *
    964  * @param Op The shl operand
    965  * @param OpVT Op's machine value value type (doesn't need to be passed, but
    966  * makes life easier.)
    967  * @return The SDNode with the entire instruction sequence
    968  */
    969 SDNode *
    970 SPUDAGToDAGISel::SelectSRLi64(SDNode *N, EVT OpVT) {
    971   SDValue Op0 = N->getOperand(0);
    972   EVT VecVT = EVT::getVectorVT(*CurDAG->getContext(),
    973                                OpVT, (128 / OpVT.getSizeInBits()));
    974   SDValue ShiftAmt = N->getOperand(1);
    975   EVT ShiftAmtVT = ShiftAmt.getValueType();
    976   SDNode *VecOp0, *Shift = 0;
    977   DebugLoc dl = N->getDebugLoc();
    978 
    979   VecOp0 = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, dl, VecVT,
    980                                   Op0, getRC(MVT::v2i64) );
    981 
    982   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(ShiftAmt)) {
    983     unsigned bytes = unsigned(CN->getZExtValue()) >> 3;
    984     unsigned bits = unsigned(CN->getZExtValue()) & 7;
    985 
    986     if (bytes > 0) {
    987       Shift =
    988         CurDAG->getMachineNode(SPU::ROTQMBYIv2i64, dl, VecVT,
    989                                SDValue(VecOp0, 0),
    990                                CurDAG->getTargetConstant(bytes, ShiftAmtVT));
    991     }
    992 
    993     if (bits > 0) {
    994       Shift =
    995         CurDAG->getMachineNode(SPU::ROTQMBIIv2i64, dl, VecVT,
    996                                SDValue((Shift != 0 ? Shift : VecOp0), 0),
    997                                CurDAG->getTargetConstant(bits, ShiftAmtVT));
    998     }
    999   } else {
   1000     SDNode *Bytes =
   1001       CurDAG->getMachineNode(SPU::ROTMIr32, dl, ShiftAmtVT,
   1002                              ShiftAmt,
   1003                              CurDAG->getTargetConstant(3, ShiftAmtVT));
   1004     SDNode *Bits =
   1005       CurDAG->getMachineNode(SPU::ANDIr32, dl, ShiftAmtVT,
   1006                              ShiftAmt,
   1007                              CurDAG->getTargetConstant(7, ShiftAmtVT));
   1008 
   1009     // Ensure that the shift amounts are negated!
   1010     Bytes = CurDAG->getMachineNode(SPU::SFIr32, dl, ShiftAmtVT,
   1011                                    SDValue(Bytes, 0),
   1012                                    CurDAG->getTargetConstant(0, ShiftAmtVT));
   1013 
   1014     Bits = CurDAG->getMachineNode(SPU::SFIr32, dl, ShiftAmtVT,
   1015                                   SDValue(Bits, 0),
   1016                                   CurDAG->getTargetConstant(0, ShiftAmtVT));
   1017 
   1018     Shift =
   1019       CurDAG->getMachineNode(SPU::ROTQMBYv2i64, dl, VecVT,
   1020                              SDValue(VecOp0, 0), SDValue(Bytes, 0));
   1021     Shift =
   1022       CurDAG->getMachineNode(SPU::ROTQMBIv2i64, dl, VecVT,
   1023                              SDValue(Shift, 0), SDValue(Bits, 0));
   1024   }
   1025 
   1026   return CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, dl,
   1027                                 OpVT, SDValue(Shift, 0), getRC(MVT::i64));
   1028 }
   1029 
   1030 /*!
   1031  * Emit the instruction sequence for i64 arithmetic right shifts.
   1032  *
   1033  * @param Op The shl operand
   1034  * @param OpVT Op's machine value value type (doesn't need to be passed, but
   1035  * makes life easier.)
   1036  * @return The SDNode with the entire instruction sequence
   1037  */
   1038 SDNode *
   1039 SPUDAGToDAGISel::SelectSRAi64(SDNode *N, EVT OpVT) {
   1040   // Promote Op0 to vector
   1041   EVT VecVT = EVT::getVectorVT(*CurDAG->getContext(),
   1042                                OpVT, (128 / OpVT.getSizeInBits()));
   1043   SDValue ShiftAmt = N->getOperand(1);
   1044   EVT ShiftAmtVT = ShiftAmt.getValueType();
   1045   DebugLoc dl = N->getDebugLoc();
   1046 
   1047   SDNode *VecOp0 =
   1048     CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, dl,
   1049                            VecVT, N->getOperand(0), getRC(MVT::v2i64));
   1050 
   1051   SDValue SignRotAmt = CurDAG->getTargetConstant(31, ShiftAmtVT);
   1052   SDNode *SignRot =
   1053     CurDAG->getMachineNode(SPU::ROTMAIv2i64_i32, dl, MVT::v2i64,
   1054                            SDValue(VecOp0, 0), SignRotAmt);
   1055   SDNode *UpperHalfSign =
   1056     CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, dl,
   1057                            MVT::i32, SDValue(SignRot, 0), getRC(MVT::i32));
   1058 
   1059   SDNode *UpperHalfSignMask =
   1060     CurDAG->getMachineNode(SPU::FSM64r32, dl, VecVT, SDValue(UpperHalfSign, 0));
   1061   SDNode *UpperLowerMask =
   1062     CurDAG->getMachineNode(SPU::FSMBIv2i64, dl, VecVT,
   1063                            CurDAG->getTargetConstant(0xff00ULL, MVT::i16));
   1064   SDNode *UpperLowerSelect =
   1065     CurDAG->getMachineNode(SPU::SELBv2i64, dl, VecVT,
   1066                            SDValue(UpperHalfSignMask, 0),
   1067                            SDValue(VecOp0, 0),
   1068                            SDValue(UpperLowerMask, 0));
   1069 
   1070   SDNode *Shift = 0;
   1071 
   1072   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(ShiftAmt)) {
   1073     unsigned bytes = unsigned(CN->getZExtValue()) >> 3;
   1074     unsigned bits = unsigned(CN->getZExtValue()) & 7;
   1075 
   1076     if (bytes > 0) {
   1077       bytes = 31 - bytes;
   1078       Shift =
   1079         CurDAG->getMachineNode(SPU::ROTQBYIv2i64, dl, VecVT,
   1080                                SDValue(UpperLowerSelect, 0),
   1081                                CurDAG->getTargetConstant(bytes, ShiftAmtVT));
   1082     }
   1083 
   1084     if (bits > 0) {
   1085       bits = 8 - bits;
   1086       Shift =
   1087         CurDAG->getMachineNode(SPU::ROTQBIIv2i64, dl, VecVT,
   1088                                SDValue((Shift != 0 ? Shift : UpperLowerSelect), 0),
   1089                                CurDAG->getTargetConstant(bits, ShiftAmtVT));
   1090     }
   1091   } else {
   1092     SDNode *NegShift =
   1093       CurDAG->getMachineNode(SPU::SFIr32, dl, ShiftAmtVT,
   1094                              ShiftAmt, CurDAG->getTargetConstant(0, ShiftAmtVT));
   1095 
   1096     Shift =
   1097       CurDAG->getMachineNode(SPU::ROTQBYBIv2i64_r32, dl, VecVT,
   1098                              SDValue(UpperLowerSelect, 0), SDValue(NegShift, 0));
   1099     Shift =
   1100       CurDAG->getMachineNode(SPU::ROTQBIv2i64, dl, VecVT,
   1101                              SDValue(Shift, 0), SDValue(NegShift, 0));
   1102   }
   1103 
   1104   return CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, dl,
   1105                                 OpVT, SDValue(Shift, 0), getRC(MVT::i64));
   1106 }
   1107 
   1108 /*!
   1109  Do the necessary magic necessary to load a i64 constant
   1110  */
   1111 SDNode *SPUDAGToDAGISel::SelectI64Constant(SDNode *N, EVT OpVT,
   1112                                            DebugLoc dl) {
   1113   ConstantSDNode *CN = cast<ConstantSDNode>(N);
   1114   return SelectI64Constant(CN->getZExtValue(), OpVT, dl);
   1115 }
   1116 
   1117 SDNode *SPUDAGToDAGISel::SelectI64Constant(uint64_t Value64, EVT OpVT,
   1118                                            DebugLoc dl) {
   1119   EVT OpVecVT = EVT::getVectorVT(*CurDAG->getContext(), OpVT, 2);
   1120   SDValue i64vec =
   1121           SPU::LowerV2I64Splat(OpVecVT, *CurDAG, Value64, dl);
   1122 
   1123   // Here's where it gets interesting, because we have to parse out the
   1124   // subtree handed back in i64vec:
   1125 
   1126   if (i64vec.getOpcode() == ISD::BITCAST) {
   1127     // The degenerate case where the upper and lower bits in the splat are
   1128     // identical:
   1129     SDValue Op0 = i64vec.getOperand(0);
   1130 
   1131     ReplaceUses(i64vec, Op0);
   1132     return CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, dl, OpVT,
   1133                                   SDValue(emitBuildVector(Op0.getNode()), 0),
   1134                                   getRC(MVT::i64));
   1135   } else if (i64vec.getOpcode() == SPUISD::SHUFB) {
   1136     SDValue lhs = i64vec.getOperand(0);
   1137     SDValue rhs = i64vec.getOperand(1);
   1138     SDValue shufmask = i64vec.getOperand(2);
   1139 
   1140     if (lhs.getOpcode() == ISD::BITCAST) {
   1141       ReplaceUses(lhs, lhs.getOperand(0));
   1142       lhs = lhs.getOperand(0);
   1143     }
   1144 
   1145     SDNode *lhsNode = (lhs.getNode()->isMachineOpcode()
   1146                        ? lhs.getNode()
   1147                        : emitBuildVector(lhs.getNode()));
   1148 
   1149     if (rhs.getOpcode() == ISD::BITCAST) {
   1150       ReplaceUses(rhs, rhs.getOperand(0));
   1151       rhs = rhs.getOperand(0);
   1152     }
   1153 
   1154     SDNode *rhsNode = (rhs.getNode()->isMachineOpcode()
   1155                        ? rhs.getNode()
   1156                        : emitBuildVector(rhs.getNode()));
   1157 
   1158     if (shufmask.getOpcode() == ISD::BITCAST) {
   1159       ReplaceUses(shufmask, shufmask.getOperand(0));
   1160       shufmask = shufmask.getOperand(0);
   1161     }
   1162 
   1163     SDNode *shufMaskNode = (shufmask.getNode()->isMachineOpcode()
   1164                             ? shufmask.getNode()
   1165                             : emitBuildVector(shufmask.getNode()));
   1166 
   1167    SDValue shufNode =
   1168             CurDAG->getNode(SPUISD::SHUFB, dl, OpVecVT,
   1169                                    SDValue(lhsNode, 0), SDValue(rhsNode, 0),
   1170                                    SDValue(shufMaskNode, 0));
   1171     HandleSDNode Dummy(shufNode);
   1172     SDNode *SN = SelectCode(Dummy.getValue().getNode());
   1173     if (SN == 0) SN = Dummy.getValue().getNode();
   1174 
   1175     return CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, dl,
   1176                                   OpVT, SDValue(SN, 0), getRC(MVT::i64));
   1177   } else if (i64vec.getOpcode() == ISD::BUILD_VECTOR) {
   1178     return CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, dl, OpVT,
   1179                                   SDValue(emitBuildVector(i64vec.getNode()), 0),
   1180                                   getRC(MVT::i64));
   1181   } else {
   1182     report_fatal_error("SPUDAGToDAGISel::SelectI64Constant: Unhandled i64vec"
   1183                       "condition");
   1184   }
   1185 }
   1186 
   1187 /// createSPUISelDag - This pass converts a legalized DAG into a
   1188 /// SPU-specific DAG, ready for instruction scheduling.
   1189 ///
   1190 FunctionPass *llvm::createSPUISelDag(SPUTargetMachine &TM) {
   1191   return new SPUDAGToDAGISel(TM);
   1192 }
   1193