Home | History | Annotate | Download | only in SelectionDAG
      1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 // This file implements integer type expansion and promotion for LegalizeTypes.
     11 // Promotion is the act of changing a computation in an illegal type into a
     12 // computation in a larger type.  For example, implementing i8 arithmetic in an
     13 // i32 register (often needed on powerpc).
     14 // Expansion is the act of changing a computation in an illegal type into a
     15 // computation in two identical registers of a smaller type.  For example,
     16 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
     17 // targets).
     18 //
     19 //===----------------------------------------------------------------------===//
     20 
     21 #include "LegalizeTypes.h"
     22 #include "llvm/DerivedTypes.h"
     23 #include "llvm/Support/ErrorHandling.h"
     24 #include "llvm/Support/raw_ostream.h"
     25 using namespace llvm;
     26 
     27 //===----------------------------------------------------------------------===//
     28 //  Integer Result Promotion
     29 //===----------------------------------------------------------------------===//
     30 
     31 /// PromoteIntegerResult - This method is called when a result of a node is
     32 /// found to be in need of promotion to a larger type.  At this point, the node
     33 /// may also have invalid operands or may have other results that need
     34 /// expansion, we just know that (at least) one result needs promotion.
     35 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
     36   DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); dbgs() << "\n");
     37   SDValue Res = SDValue();
     38 
     39   // See if the target wants to custom expand this node.
     40   if (CustomLowerNode(N, N->getValueType(ResNo), true))
     41     return;
     42 
     43   switch (N->getOpcode()) {
     44   default:
     45 #ifndef NDEBUG
     46     dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
     47     N->dump(&DAG); dbgs() << "\n";
     48 #endif
     49     llvm_unreachable("Do not know how to promote this operator!");
     50   case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
     51   case ISD::AssertSext:  Res = PromoteIntRes_AssertSext(N); break;
     52   case ISD::AssertZext:  Res = PromoteIntRes_AssertZext(N); break;
     53   case ISD::BITCAST:     Res = PromoteIntRes_BITCAST(N); break;
     54   case ISD::BSWAP:       Res = PromoteIntRes_BSWAP(N); break;
     55   case ISD::BUILD_PAIR:  Res = PromoteIntRes_BUILD_PAIR(N); break;
     56   case ISD::Constant:    Res = PromoteIntRes_Constant(N); break;
     57   case ISD::CONVERT_RNDSAT:
     58                          Res = PromoteIntRes_CONVERT_RNDSAT(N); break;
     59   case ISD::CTLZ_ZERO_UNDEF:
     60   case ISD::CTLZ:        Res = PromoteIntRes_CTLZ(N); break;
     61   case ISD::CTPOP:       Res = PromoteIntRes_CTPOP(N); break;
     62   case ISD::CTTZ_ZERO_UNDEF:
     63   case ISD::CTTZ:        Res = PromoteIntRes_CTTZ(N); break;
     64   case ISD::EXTRACT_VECTOR_ELT:
     65                          Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
     66   case ISD::LOAD:        Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N));break;
     67   case ISD::SELECT:      Res = PromoteIntRes_SELECT(N); break;
     68   case ISD::VSELECT:     Res = PromoteIntRes_VSELECT(N); break;
     69   case ISD::SELECT_CC:   Res = PromoteIntRes_SELECT_CC(N); break;
     70   case ISD::SETCC:       Res = PromoteIntRes_SETCC(N); break;
     71   case ISD::SHL:         Res = PromoteIntRes_SHL(N); break;
     72   case ISD::SIGN_EXTEND_INREG:
     73                          Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
     74   case ISD::SRA:         Res = PromoteIntRes_SRA(N); break;
     75   case ISD::SRL:         Res = PromoteIntRes_SRL(N); break;
     76   case ISD::TRUNCATE:    Res = PromoteIntRes_TRUNCATE(N); break;
     77   case ISD::UNDEF:       Res = PromoteIntRes_UNDEF(N); break;
     78   case ISD::VAARG:       Res = PromoteIntRes_VAARG(N); break;
     79 
     80   case ISD::EXTRACT_SUBVECTOR:
     81                          Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
     82   case ISD::VECTOR_SHUFFLE:
     83                          Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
     84   case ISD::INSERT_VECTOR_ELT:
     85                          Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
     86   case ISD::BUILD_VECTOR:
     87                          Res = PromoteIntRes_BUILD_VECTOR(N); break;
     88   case ISD::SCALAR_TO_VECTOR:
     89                          Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
     90   case ISD::CONCAT_VECTORS:
     91                          Res = PromoteIntRes_CONCAT_VECTORS(N); break;
     92 
     93   case ISD::SIGN_EXTEND:
     94   case ISD::ZERO_EXTEND:
     95   case ISD::ANY_EXTEND:  Res = PromoteIntRes_INT_EXTEND(N); break;
     96 
     97   case ISD::FP_TO_SINT:
     98   case ISD::FP_TO_UINT:  Res = PromoteIntRes_FP_TO_XINT(N); break;
     99 
    100   case ISD::FP32_TO_FP16:Res = PromoteIntRes_FP32_TO_FP16(N); break;
    101 
    102   case ISD::AND:
    103   case ISD::OR:
    104   case ISD::XOR:
    105   case ISD::ADD:
    106   case ISD::SUB:
    107   case ISD::MUL:         Res = PromoteIntRes_SimpleIntBinOp(N); break;
    108 
    109   case ISD::SDIV:
    110   case ISD::SREM:        Res = PromoteIntRes_SDIV(N); break;
    111 
    112   case ISD::UDIV:
    113   case ISD::UREM:        Res = PromoteIntRes_UDIV(N); break;
    114 
    115   case ISD::SADDO:
    116   case ISD::SSUBO:       Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
    117   case ISD::UADDO:
    118   case ISD::USUBO:       Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
    119   case ISD::SMULO:
    120   case ISD::UMULO:       Res = PromoteIntRes_XMULO(N, ResNo); break;
    121 
    122   case ISD::ATOMIC_LOAD:
    123     Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
    124 
    125   case ISD::ATOMIC_LOAD_ADD:
    126   case ISD::ATOMIC_LOAD_SUB:
    127   case ISD::ATOMIC_LOAD_AND:
    128   case ISD::ATOMIC_LOAD_OR:
    129   case ISD::ATOMIC_LOAD_XOR:
    130   case ISD::ATOMIC_LOAD_NAND:
    131   case ISD::ATOMIC_LOAD_MIN:
    132   case ISD::ATOMIC_LOAD_MAX:
    133   case ISD::ATOMIC_LOAD_UMIN:
    134   case ISD::ATOMIC_LOAD_UMAX:
    135   case ISD::ATOMIC_SWAP:
    136     Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
    137 
    138   case ISD::ATOMIC_CMP_SWAP:
    139     Res = PromoteIntRes_Atomic2(cast<AtomicSDNode>(N)); break;
    140   }
    141 
    142   // If the result is null then the sub-method took care of registering it.
    143   if (Res.getNode())
    144     SetPromotedInteger(SDValue(N, ResNo), Res);
    145 }
    146 
    147 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
    148                                                      unsigned ResNo) {
    149   SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
    150   return GetPromotedInteger(Op);
    151 }
    152 
    153 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
    154   // Sign-extend the new bits, and continue the assertion.
    155   SDValue Op = SExtPromotedInteger(N->getOperand(0));
    156   return DAG.getNode(ISD::AssertSext, N->getDebugLoc(),
    157                      Op.getValueType(), Op, N->getOperand(1));
    158 }
    159 
    160 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
    161   // Zero the new bits, and continue the assertion.
    162   SDValue Op = ZExtPromotedInteger(N->getOperand(0));
    163   return DAG.getNode(ISD::AssertZext, N->getDebugLoc(),
    164                      Op.getValueType(), Op, N->getOperand(1));
    165 }
    166 
    167 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
    168   EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    169   SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(),
    170                               N->getMemoryVT(), ResVT,
    171                               N->getChain(), N->getBasePtr(),
    172                               N->getMemOperand(), N->getOrdering(),
    173                               N->getSynchScope());
    174   // Legalized the chain result - switch anything that used the old chain to
    175   // use the new one.
    176   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
    177   return Res;
    178 }
    179 
    180 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
    181   SDValue Op2 = GetPromotedInteger(N->getOperand(2));
    182   SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(),
    183                               N->getMemoryVT(),
    184                               N->getChain(), N->getBasePtr(),
    185                               Op2, N->getMemOperand(), N->getOrdering(),
    186                               N->getSynchScope());
    187   // Legalized the chain result - switch anything that used the old chain to
    188   // use the new one.
    189   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
    190   return Res;
    191 }
    192 
    193 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic2(AtomicSDNode *N) {
    194   SDValue Op2 = GetPromotedInteger(N->getOperand(2));
    195   SDValue Op3 = GetPromotedInteger(N->getOperand(3));
    196   SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(),
    197                               N->getMemoryVT(), N->getChain(), N->getBasePtr(),
    198                               Op2, Op3, N->getMemOperand(), N->getOrdering(),
    199                               N->getSynchScope());
    200   // Legalized the chain result - switch anything that used the old chain to
    201   // use the new one.
    202   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
    203   return Res;
    204 }
    205 
    206 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
    207   SDValue InOp = N->getOperand(0);
    208   EVT InVT = InOp.getValueType();
    209   EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
    210   EVT OutVT = N->getValueType(0);
    211   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
    212   DebugLoc dl = N->getDebugLoc();
    213 
    214   switch (getTypeAction(InVT)) {
    215   case TargetLowering::TypeLegal:
    216     break;
    217   case TargetLowering::TypePromoteInteger:
    218     if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
    219       // The input promotes to the same size.  Convert the promoted value.
    220       return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
    221     break;
    222   case TargetLowering::TypeSoftenFloat:
    223     // Promote the integer operand by hand.
    224     return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
    225   case TargetLowering::TypeExpandInteger:
    226   case TargetLowering::TypeExpandFloat:
    227     break;
    228   case TargetLowering::TypeScalarizeVector:
    229     // Convert the element to an integer and promote it by hand.
    230     if (!NOutVT.isVector())
    231       return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
    232                          BitConvertToInteger(GetScalarizedVector(InOp)));
    233     break;
    234   case TargetLowering::TypeSplitVector: {
    235     // For example, i32 = BITCAST v2i16 on alpha.  Convert the split
    236     // pieces of the input into integers and reassemble in the final type.
    237     SDValue Lo, Hi;
    238     GetSplitVector(N->getOperand(0), Lo, Hi);
    239     Lo = BitConvertToInteger(Lo);
    240     Hi = BitConvertToInteger(Hi);
    241 
    242     if (TLI.isBigEndian())
    243       std::swap(Lo, Hi);
    244 
    245     InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
    246                        EVT::getIntegerVT(*DAG.getContext(),
    247                                          NOutVT.getSizeInBits()),
    248                        JoinIntegers(Lo, Hi));
    249     return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
    250   }
    251   case TargetLowering::TypeWidenVector:
    252     // The input is widened to the same size. Convert to the widened value.
    253     // Make sure that the outgoing value is not a vector, because this would
    254     // make us bitcast between two vectors which are legalized in different ways.
    255     if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector())
    256       return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
    257   }
    258 
    259   return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
    260                      CreateStackStoreLoad(InOp, OutVT));
    261 }
    262 
    263 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
    264   SDValue Op = GetPromotedInteger(N->getOperand(0));
    265   EVT OVT = N->getValueType(0);
    266   EVT NVT = Op.getValueType();
    267   DebugLoc dl = N->getDebugLoc();
    268 
    269   unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
    270   return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
    271                      DAG.getConstant(DiffBits, TLI.getPointerTy()));
    272 }
    273 
    274 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
    275   // The pair element type may be legal, or may not promote to the same type as
    276   // the result, for example i14 = BUILD_PAIR (i7, i7).  Handle all cases.
    277   return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(),
    278                      TLI.getTypeToTransformTo(*DAG.getContext(),
    279                      N->getValueType(0)), JoinIntegers(N->getOperand(0),
    280                      N->getOperand(1)));
    281 }
    282 
    283 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
    284   EVT VT = N->getValueType(0);
    285   // FIXME there is no actual debug info here
    286   DebugLoc dl = N->getDebugLoc();
    287   // Zero extend things like i1, sign extend everything else.  It shouldn't
    288   // matter in theory which one we pick, but this tends to give better code?
    289   unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
    290   SDValue Result = DAG.getNode(Opc, dl,
    291                                TLI.getTypeToTransformTo(*DAG.getContext(), VT),
    292                                SDValue(N, 0));
    293   assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
    294   return Result;
    295 }
    296 
    297 SDValue DAGTypeLegalizer::PromoteIntRes_CONVERT_RNDSAT(SDNode *N) {
    298   ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode();
    299   assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU ||
    300            CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU ||
    301            CvtCode == ISD::CVT_SF || CvtCode == ISD::CVT_UF) &&
    302           "can only promote integers");
    303   EVT OutVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    304   return DAG.getConvertRndSat(OutVT, N->getDebugLoc(), N->getOperand(0),
    305                               N->getOperand(1), N->getOperand(2),
    306                               N->getOperand(3), N->getOperand(4), CvtCode);
    307 }
    308 
    309 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
    310   // Zero extend to the promoted type and do the count there.
    311   SDValue Op = ZExtPromotedInteger(N->getOperand(0));
    312   DebugLoc dl = N->getDebugLoc();
    313   EVT OVT = N->getValueType(0);
    314   EVT NVT = Op.getValueType();
    315   Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
    316   // Subtract off the extra leading bits in the bigger type.
    317   return DAG.getNode(ISD::SUB, dl, NVT, Op,
    318                      DAG.getConstant(NVT.getSizeInBits() -
    319                                      OVT.getSizeInBits(), NVT));
    320 }
    321 
    322 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
    323   // Zero extend to the promoted type and do the count there.
    324   SDValue Op = ZExtPromotedInteger(N->getOperand(0));
    325   return DAG.getNode(ISD::CTPOP, N->getDebugLoc(), Op.getValueType(), Op);
    326 }
    327 
    328 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
    329   SDValue Op = GetPromotedInteger(N->getOperand(0));
    330   EVT OVT = N->getValueType(0);
    331   EVT NVT = Op.getValueType();
    332   DebugLoc dl = N->getDebugLoc();
    333   if (N->getOpcode() == ISD::CTTZ) {
    334     // The count is the same in the promoted type except if the original
    335     // value was zero.  This can be handled by setting the bit just off
    336     // the top of the original type.
    337     APInt TopBit(NVT.getSizeInBits(), 0);
    338     TopBit.setBit(OVT.getSizeInBits());
    339     Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, NVT));
    340   }
    341   return DAG.getNode(N->getOpcode(), dl, NVT, Op);
    342 }
    343 
    344 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
    345   DebugLoc dl = N->getDebugLoc();
    346   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    347   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0),
    348                      N->getOperand(1));
    349 }
    350 
    351 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
    352   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    353   unsigned NewOpc = N->getOpcode();
    354   DebugLoc dl = N->getDebugLoc();
    355 
    356   // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
    357   // not Legal, check to see if we can use FP_TO_SINT instead.  (If both UINT
    358   // and SINT conversions are Custom, there is no way to tell which is
    359   // preferable. We choose SINT because that's the right thing on PPC.)
    360   if (N->getOpcode() == ISD::FP_TO_UINT &&
    361       !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
    362       TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
    363     NewOpc = ISD::FP_TO_SINT;
    364 
    365   SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
    366 
    367   // Assert that the converted value fits in the original type.  If it doesn't
    368   // (eg: because the value being converted is too big), then the result of the
    369   // original operation was undefined anyway, so the assert is still correct.
    370   return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ?
    371                      ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
    372                      DAG.getValueType(N->getValueType(0).getScalarType()));
    373 }
    374 
    375 SDValue DAGTypeLegalizer::PromoteIntRes_FP32_TO_FP16(SDNode *N) {
    376   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    377   DebugLoc dl = N->getDebugLoc();
    378 
    379   SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
    380 
    381   return DAG.getNode(ISD::AssertZext, dl,
    382                      NVT, Res, DAG.getValueType(N->getValueType(0)));
    383 }
    384 
    385 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
    386   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    387   DebugLoc dl = N->getDebugLoc();
    388 
    389   if (getTypeAction(N->getOperand(0).getValueType())
    390       == TargetLowering::TypePromoteInteger) {
    391     SDValue Res = GetPromotedInteger(N->getOperand(0));
    392     assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
    393 
    394     // If the result and operand types are the same after promotion, simplify
    395     // to an in-register extension.
    396     if (NVT == Res.getValueType()) {
    397       // The high bits are not guaranteed to be anything.  Insert an extend.
    398       if (N->getOpcode() == ISD::SIGN_EXTEND)
    399         return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
    400                            DAG.getValueType(N->getOperand(0).getValueType()));
    401       if (N->getOpcode() == ISD::ZERO_EXTEND)
    402         return DAG.getZeroExtendInReg(Res, dl,
    403                       N->getOperand(0).getValueType().getScalarType());
    404       assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
    405       return Res;
    406     }
    407   }
    408 
    409   // Otherwise, just extend the original operand all the way to the larger type.
    410   return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
    411 }
    412 
    413 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
    414   assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
    415   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    416   ISD::LoadExtType ExtType =
    417     ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
    418   DebugLoc dl = N->getDebugLoc();
    419   SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
    420                                N->getPointerInfo(),
    421                                N->getMemoryVT(), N->isVolatile(),
    422                                N->isNonTemporal(), N->getAlignment());
    423 
    424   // Legalized the chain result - switch anything that used the old chain to
    425   // use the new one.
    426   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
    427   return Res;
    428 }
    429 
    430 /// Promote the overflow flag of an overflowing arithmetic node.
    431 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
    432   // Simply change the return type of the boolean result.
    433   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
    434   EVT ValueVTs[] = { N->getValueType(0), NVT };
    435   SDValue Ops[] = { N->getOperand(0), N->getOperand(1) };
    436   SDValue Res = DAG.getNode(N->getOpcode(), N->getDebugLoc(),
    437                             DAG.getVTList(ValueVTs, 2), Ops, 2);
    438 
    439   // Modified the sum result - switch anything that used the old sum to use
    440   // the new one.
    441   ReplaceValueWith(SDValue(N, 0), Res);
    442 
    443   return SDValue(Res.getNode(), 1);
    444 }
    445 
    446 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
    447   if (ResNo == 1)
    448     return PromoteIntRes_Overflow(N);
    449 
    450   // The operation overflowed iff the result in the larger type is not the
    451   // sign extension of its truncation to the original type.
    452   SDValue LHS = SExtPromotedInteger(N->getOperand(0));
    453   SDValue RHS = SExtPromotedInteger(N->getOperand(1));
    454   EVT OVT = N->getOperand(0).getValueType();
    455   EVT NVT = LHS.getValueType();
    456   DebugLoc dl = N->getDebugLoc();
    457 
    458   // Do the arithmetic in the larger type.
    459   unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
    460   SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
    461 
    462   // Calculate the overflow flag: sign extend the arithmetic result from
    463   // the original type.
    464   SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
    465                             DAG.getValueType(OVT));
    466   // Overflowed if and only if this is not equal to Res.
    467   Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
    468 
    469   // Use the calculated overflow everywhere.
    470   ReplaceValueWith(SDValue(N, 1), Ofl);
    471 
    472   return Res;
    473 }
    474 
    475 SDValue DAGTypeLegalizer::PromoteIntRes_SDIV(SDNode *N) {
    476   // Sign extend the input.
    477   SDValue LHS = SExtPromotedInteger(N->getOperand(0));
    478   SDValue RHS = SExtPromotedInteger(N->getOperand(1));
    479   return DAG.getNode(N->getOpcode(), N->getDebugLoc(),
    480                      LHS.getValueType(), LHS, RHS);
    481 }
    482 
    483 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
    484   SDValue LHS = GetPromotedInteger(N->getOperand(1));
    485   SDValue RHS = GetPromotedInteger(N->getOperand(2));
    486   return DAG.getNode(ISD::SELECT, N->getDebugLoc(),
    487                      LHS.getValueType(), N->getOperand(0),LHS,RHS);
    488 }
    489 
    490 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
    491   SDValue Mask = N->getOperand(0);
    492   EVT OpTy = N->getOperand(1).getValueType();
    493 
    494   // Promote all the way up to the canonical SetCC type.
    495   Mask = PromoteTargetBoolean(Mask, TLI.getSetCCResultType(OpTy));
    496   SDValue LHS = GetPromotedInteger(N->getOperand(1));
    497   SDValue RHS = GetPromotedInteger(N->getOperand(2));
    498   return DAG.getNode(ISD::VSELECT, N->getDebugLoc(),
    499                      LHS.getValueType(), Mask, LHS, RHS);
    500 }
    501 
    502 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
    503   SDValue LHS = GetPromotedInteger(N->getOperand(2));
    504   SDValue RHS = GetPromotedInteger(N->getOperand(3));
    505   return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(),
    506                      LHS.getValueType(), N->getOperand(0),
    507                      N->getOperand(1), LHS, RHS, N->getOperand(4));
    508 }
    509 
    510 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
    511   EVT SVT = TLI.getSetCCResultType(N->getOperand(0).getValueType());
    512 
    513   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    514 
    515   // Only use the result of getSetCCResultType if it is legal,
    516   // otherwise just use the promoted result type (NVT).
    517   if (!TLI.isTypeLegal(SVT))
    518       SVT = NVT;
    519 
    520   DebugLoc dl = N->getDebugLoc();
    521   assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
    522          "Vector compare must return a vector result!");
    523 
    524   // Get the SETCC result using the canonical SETCC type.
    525   SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
    526                               N->getOperand(1), N->getOperand(2));
    527 
    528   assert(NVT.bitsLE(SVT) && "Integer type overpromoted?");
    529   // Convert to the expected type.
    530   return DAG.getNode(ISD::TRUNCATE, dl, NVT, SetCC);
    531 }
    532 
    533 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
    534   return DAG.getNode(ISD::SHL, N->getDebugLoc(),
    535                 TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)),
    536                      GetPromotedInteger(N->getOperand(0)), N->getOperand(1));
    537 }
    538 
    539 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
    540   SDValue Op = GetPromotedInteger(N->getOperand(0));
    541   return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(),
    542                      Op.getValueType(), Op, N->getOperand(1));
    543 }
    544 
    545 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
    546   // The input may have strange things in the top bits of the registers, but
    547   // these operations don't care.  They may have weird bits going out, but
    548   // that too is okay if they are integer operations.
    549   SDValue LHS = GetPromotedInteger(N->getOperand(0));
    550   SDValue RHS = GetPromotedInteger(N->getOperand(1));
    551   return DAG.getNode(N->getOpcode(), N->getDebugLoc(),
    552                     LHS.getValueType(), LHS, RHS);
    553 }
    554 
    555 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
    556   // The input value must be properly sign extended.
    557   SDValue Res = SExtPromotedInteger(N->getOperand(0));
    558   return DAG.getNode(ISD::SRA, N->getDebugLoc(),
    559                      Res.getValueType(), Res, N->getOperand(1));
    560 }
    561 
    562 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
    563   // The input value must be properly zero extended.
    564   EVT VT = N->getValueType(0);
    565   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    566   SDValue Res = ZExtPromotedInteger(N->getOperand(0));
    567   return DAG.getNode(ISD::SRL, N->getDebugLoc(), NVT, Res, N->getOperand(1));
    568 }
    569 
    570 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
    571   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    572   SDValue Res;
    573   SDValue InOp = N->getOperand(0);
    574   DebugLoc dl = N->getDebugLoc();
    575 
    576   switch (getTypeAction(InOp.getValueType())) {
    577   default: llvm_unreachable("Unknown type action!");
    578   case TargetLowering::TypeLegal:
    579   case TargetLowering::TypeExpandInteger:
    580     Res = InOp;
    581     break;
    582   case TargetLowering::TypePromoteInteger:
    583     Res = GetPromotedInteger(InOp);
    584     break;
    585   case TargetLowering::TypeSplitVector:
    586     EVT InVT = InOp.getValueType();
    587     assert(InVT.isVector() && "Cannot split scalar types");
    588     unsigned NumElts = InVT.getVectorNumElements();
    589     assert(NumElts == NVT.getVectorNumElements() &&
    590            "Dst and Src must have the same number of elements");
    591     EVT EltVT = InVT.getScalarType();
    592     assert(isPowerOf2_32(NumElts) &&
    593            "Promoted vector type must be a power of two");
    594 
    595     EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts/2);
    596     EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
    597                                    NumElts/2);
    598 
    599     SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HalfVT, InOp,
    600                                DAG.getIntPtrConstant(0));
    601     SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HalfVT, InOp,
    602                                DAG.getIntPtrConstant(NumElts/2));
    603     EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
    604     EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
    605 
    606     return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
    607   }
    608 
    609   // Truncate to NVT instead of VT
    610   return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
    611 }
    612 
    613 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
    614   if (ResNo == 1)
    615     return PromoteIntRes_Overflow(N);
    616 
    617   // The operation overflowed iff the result in the larger type is not the
    618   // zero extension of its truncation to the original type.
    619   SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
    620   SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
    621   EVT OVT = N->getOperand(0).getValueType();
    622   EVT NVT = LHS.getValueType();
    623   DebugLoc dl = N->getDebugLoc();
    624 
    625   // Do the arithmetic in the larger type.
    626   unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
    627   SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
    628 
    629   // Calculate the overflow flag: zero extend the arithmetic result from
    630   // the original type.
    631   SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
    632   // Overflowed if and only if this is not equal to Res.
    633   Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
    634 
    635   // Use the calculated overflow everywhere.
    636   ReplaceValueWith(SDValue(N, 1), Ofl);
    637 
    638   return Res;
    639 }
    640 
    641 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
    642   // Promote the overflow bit trivially.
    643   if (ResNo == 1)
    644     return PromoteIntRes_Overflow(N);
    645 
    646   SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
    647   DebugLoc DL = N->getDebugLoc();
    648   EVT SmallVT = LHS.getValueType();
    649 
    650   // To determine if the result overflowed in a larger type, we extend the
    651   // input to the larger type, do the multiply, then check the high bits of
    652   // the result to see if the overflow happened.
    653   if (N->getOpcode() == ISD::SMULO) {
    654     LHS = SExtPromotedInteger(LHS);
    655     RHS = SExtPromotedInteger(RHS);
    656   } else {
    657     LHS = ZExtPromotedInteger(LHS);
    658     RHS = ZExtPromotedInteger(RHS);
    659   }
    660   SDValue Mul = DAG.getNode(ISD::MUL, DL, LHS.getValueType(), LHS, RHS);
    661 
    662   // Overflow occurred iff the high part of the result does not
    663   // zero/sign-extend the low part.
    664   SDValue Overflow;
    665   if (N->getOpcode() == ISD::UMULO) {
    666     // Unsigned overflow occurred iff the high part is non-zero.
    667     SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
    668                              DAG.getIntPtrConstant(SmallVT.getSizeInBits()));
    669     Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
    670                             DAG.getConstant(0, Hi.getValueType()), ISD::SETNE);
    671   } else {
    672     // Signed overflow occurred iff the high part does not sign extend the low.
    673     SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
    674                                Mul, DAG.getValueType(SmallVT));
    675     Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
    676   }
    677 
    678   // Use the calculated overflow everywhere.
    679   ReplaceValueWith(SDValue(N, 1), Overflow);
    680   return Mul;
    681 }
    682 
    683 SDValue DAGTypeLegalizer::PromoteIntRes_UDIV(SDNode *N) {
    684   // Zero extend the input.
    685   SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
    686   SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
    687   return DAG.getNode(N->getOpcode(), N->getDebugLoc(),
    688                      LHS.getValueType(), LHS, RHS);
    689 }
    690 
    691 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
    692   return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
    693                                                N->getValueType(0)));
    694 }
    695 
    696 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
    697   SDValue Chain = N->getOperand(0); // Get the chain.
    698   SDValue Ptr = N->getOperand(1); // Get the pointer.
    699   EVT VT = N->getValueType(0);
    700   DebugLoc dl = N->getDebugLoc();
    701 
    702   EVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
    703   unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
    704   // The argument is passed as NumRegs registers of type RegVT.
    705 
    706   SmallVector<SDValue, 8> Parts(NumRegs);
    707   for (unsigned i = 0; i < NumRegs; ++i) {
    708     Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
    709                             N->getConstantOperandVal(3));
    710     Chain = Parts[i].getValue(1);
    711   }
    712 
    713   // Handle endianness of the load.
    714   if (TLI.isBigEndian())
    715     std::reverse(Parts.begin(), Parts.end());
    716 
    717   // Assemble the parts in the promoted type.
    718   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    719   SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
    720   for (unsigned i = 1; i < NumRegs; ++i) {
    721     SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
    722     // Shift it to the right position and "or" it in.
    723     Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
    724                        DAG.getConstant(i * RegVT.getSizeInBits(),
    725                                        TLI.getPointerTy()));
    726     Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
    727   }
    728 
    729   // Modified the chain result - switch anything that used the old chain to
    730   // use the new one.
    731   ReplaceValueWith(SDValue(N, 1), Chain);
    732 
    733   return Res;
    734 }
    735 
    736 //===----------------------------------------------------------------------===//
    737 //  Integer Operand Promotion
    738 //===----------------------------------------------------------------------===//
    739 
    740 /// PromoteIntegerOperand - This method is called when the specified operand of
    741 /// the specified node is found to need promotion.  At this point, all of the
    742 /// result types of the node are known to be legal, but other operands of the
    743 /// node may need promotion or expansion as well as the specified one.
    744 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
    745   DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); dbgs() << "\n");
    746   SDValue Res = SDValue();
    747 
    748   if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
    749     return false;
    750 
    751   switch (N->getOpcode()) {
    752     default:
    753   #ifndef NDEBUG
    754     dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
    755     N->dump(&DAG); dbgs() << "\n";
    756   #endif
    757     llvm_unreachable("Do not know how to promote this operator's operand!");
    758 
    759   case ISD::ANY_EXTEND:   Res = PromoteIntOp_ANY_EXTEND(N); break;
    760   case ISD::ATOMIC_STORE:
    761     Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
    762     break;
    763   case ISD::BITCAST:      Res = PromoteIntOp_BITCAST(N); break;
    764   case ISD::BR_CC:        Res = PromoteIntOp_BR_CC(N, OpNo); break;
    765   case ISD::BRCOND:       Res = PromoteIntOp_BRCOND(N, OpNo); break;
    766   case ISD::BUILD_PAIR:   Res = PromoteIntOp_BUILD_PAIR(N); break;
    767   case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
    768   case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
    769   case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
    770   case ISD::CONVERT_RNDSAT:
    771                           Res = PromoteIntOp_CONVERT_RNDSAT(N); break;
    772   case ISD::INSERT_VECTOR_ELT:
    773                           Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
    774   case ISD::MEMBARRIER:   Res = PromoteIntOp_MEMBARRIER(N); break;
    775   case ISD::SCALAR_TO_VECTOR:
    776                           Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
    777   case ISD::VSELECT:
    778   case ISD::SELECT:       Res = PromoteIntOp_SELECT(N, OpNo); break;
    779   case ISD::SELECT_CC:    Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
    780   case ISD::SETCC:        Res = PromoteIntOp_SETCC(N, OpNo); break;
    781   case ISD::SIGN_EXTEND:  Res = PromoteIntOp_SIGN_EXTEND(N); break;
    782   case ISD::SINT_TO_FP:   Res = PromoteIntOp_SINT_TO_FP(N); break;
    783   case ISD::STORE:        Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
    784                                                    OpNo); break;
    785   case ISD::TRUNCATE:     Res = PromoteIntOp_TRUNCATE(N); break;
    786   case ISD::FP16_TO_FP32:
    787   case ISD::UINT_TO_FP:   Res = PromoteIntOp_UINT_TO_FP(N); break;
    788   case ISD::ZERO_EXTEND:  Res = PromoteIntOp_ZERO_EXTEND(N); break;
    789 
    790   case ISD::SHL:
    791   case ISD::SRA:
    792   case ISD::SRL:
    793   case ISD::ROTL:
    794   case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
    795   }
    796 
    797   // If the result is null, the sub-method took care of registering results etc.
    798   if (!Res.getNode()) return false;
    799 
    800   // If the result is N, the sub-method updated N in place.  Tell the legalizer
    801   // core about this.
    802   if (Res.getNode() == N)
    803     return true;
    804 
    805   assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
    806          "Invalid operand expansion");
    807 
    808   ReplaceValueWith(SDValue(N, 0), Res);
    809   return false;
    810 }
    811 
    812 /// PromoteSetCCOperands - Promote the operands of a comparison.  This code is
    813 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
    814 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
    815                                             ISD::CondCode CCCode) {
    816   // We have to insert explicit sign or zero extends.  Note that we could
    817   // insert sign extends for ALL conditions, but zero extend is cheaper on
    818   // many machines (an AND instead of two shifts), so prefer it.
    819   switch (CCCode) {
    820   default: llvm_unreachable("Unknown integer comparison!");
    821   case ISD::SETEQ:
    822   case ISD::SETNE:
    823   case ISD::SETUGE:
    824   case ISD::SETUGT:
    825   case ISD::SETULE:
    826   case ISD::SETULT:
    827     // ALL of these operations will work if we either sign or zero extend
    828     // the operands (including the unsigned comparisons!).  Zero extend is
    829     // usually a simpler/cheaper operation, so prefer it.
    830     NewLHS = ZExtPromotedInteger(NewLHS);
    831     NewRHS = ZExtPromotedInteger(NewRHS);
    832     break;
    833   case ISD::SETGE:
    834   case ISD::SETGT:
    835   case ISD::SETLT:
    836   case ISD::SETLE:
    837     NewLHS = SExtPromotedInteger(NewLHS);
    838     NewRHS = SExtPromotedInteger(NewRHS);
    839     break;
    840   }
    841 }
    842 
    843 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
    844   SDValue Op = GetPromotedInteger(N->getOperand(0));
    845   return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), N->getValueType(0), Op);
    846 }
    847 
    848 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
    849   SDValue Op2 = GetPromotedInteger(N->getOperand(2));
    850   return DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), N->getMemoryVT(),
    851                        N->getChain(), N->getBasePtr(), Op2, N->getMemOperand(),
    852                        N->getOrdering(), N->getSynchScope());
    853 }
    854 
    855 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
    856   // This should only occur in unusual situations like bitcasting to an
    857   // x86_fp80, so just turn it into a store+load
    858   return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
    859 }
    860 
    861 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
    862   assert(OpNo == 2 && "Don't know how to promote this operand!");
    863 
    864   SDValue LHS = N->getOperand(2);
    865   SDValue RHS = N->getOperand(3);
    866   PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
    867 
    868   // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
    869   // legal types.
    870   return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
    871                                 N->getOperand(1), LHS, RHS, N->getOperand(4)),
    872                  0);
    873 }
    874 
    875 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
    876   assert(OpNo == 1 && "only know how to promote condition");
    877 
    878   // Promote all the way up to the canonical SetCC type.
    879   EVT SVT = TLI.getSetCCResultType(MVT::Other);
    880   SDValue Cond = PromoteTargetBoolean(N->getOperand(1), SVT);
    881 
    882   // The chain (Op#0) and basic block destination (Op#2) are always legal types.
    883   return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
    884                                         N->getOperand(2)), 0);
    885 }
    886 
    887 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
    888   // Since the result type is legal, the operands must promote to it.
    889   EVT OVT = N->getOperand(0).getValueType();
    890   SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
    891   SDValue Hi = GetPromotedInteger(N->getOperand(1));
    892   assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
    893   DebugLoc dl = N->getDebugLoc();
    894 
    895   Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
    896                    DAG.getConstant(OVT.getSizeInBits(), TLI.getPointerTy()));
    897   return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
    898 }
    899 
    900 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
    901   // The vector type is legal but the element type is not.  This implies
    902   // that the vector is a power-of-two in length and that the element
    903   // type does not have a strange size (eg: it is not i1).
    904   EVT VecVT = N->getValueType(0);
    905   unsigned NumElts = VecVT.getVectorNumElements();
    906   assert(!(NumElts & 1) && "Legal vector of one illegal element?");
    907 
    908   // Promote the inserted value.  The type does not need to match the
    909   // vector element type.  Check that any extra bits introduced will be
    910   // truncated away.
    911   assert(N->getOperand(0).getValueType().getSizeInBits() >=
    912          N->getValueType(0).getVectorElementType().getSizeInBits() &&
    913          "Type of inserted value narrower than vector element type!");
    914 
    915   SmallVector<SDValue, 16> NewOps;
    916   for (unsigned i = 0; i < NumElts; ++i)
    917     NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
    918 
    919   return SDValue(DAG.UpdateNodeOperands(N, &NewOps[0], NumElts), 0);
    920 }
    921 
    922 SDValue DAGTypeLegalizer::PromoteIntOp_CONVERT_RNDSAT(SDNode *N) {
    923   ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode();
    924   assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU ||
    925            CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU ||
    926            CvtCode == ISD::CVT_FS || CvtCode == ISD::CVT_FU) &&
    927            "can only promote integer arguments");
    928   SDValue InOp = GetPromotedInteger(N->getOperand(0));
    929   return DAG.getConvertRndSat(N->getValueType(0), N->getDebugLoc(), InOp,
    930                               N->getOperand(1), N->getOperand(2),
    931                               N->getOperand(3), N->getOperand(4), CvtCode);
    932 }
    933 
    934 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
    935                                                          unsigned OpNo) {
    936   if (OpNo == 1) {
    937     // Promote the inserted value.  This is valid because the type does not
    938     // have to match the vector element type.
    939 
    940     // Check that any extra bits introduced will be truncated away.
    941     assert(N->getOperand(1).getValueType().getSizeInBits() >=
    942            N->getValueType(0).getVectorElementType().getSizeInBits() &&
    943            "Type of inserted value narrower than vector element type!");
    944     return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
    945                                   GetPromotedInteger(N->getOperand(1)),
    946                                   N->getOperand(2)),
    947                    0);
    948   }
    949 
    950   assert(OpNo == 2 && "Different operand and result vector types?");
    951 
    952   // Promote the index.
    953   SDValue Idx = ZExtPromotedInteger(N->getOperand(2));
    954   return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
    955                                 N->getOperand(1), Idx), 0);
    956 }
    957 
    958 SDValue DAGTypeLegalizer::PromoteIntOp_MEMBARRIER(SDNode *N) {
    959   SDValue NewOps[6];
    960   DebugLoc dl = N->getDebugLoc();
    961   NewOps[0] = N->getOperand(0);
    962   for (unsigned i = 1; i < array_lengthof(NewOps); ++i) {
    963     SDValue Flag = GetPromotedInteger(N->getOperand(i));
    964     NewOps[i] = DAG.getZeroExtendInReg(Flag, dl, MVT::i1);
    965   }
    966   return SDValue(DAG.UpdateNodeOperands(N, NewOps, array_lengthof(NewOps)), 0);
    967 }
    968 
    969 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
    970   // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
    971   // the operand in place.
    972   return SDValue(DAG.UpdateNodeOperands(N,
    973                                 GetPromotedInteger(N->getOperand(0))), 0);
    974 }
    975 
    976 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
    977   assert(OpNo == 0 && "Only know how to promote the condition!");
    978   SDValue Cond = N->getOperand(0);
    979   EVT OpTy = N->getOperand(1).getValueType();
    980 
    981   // Promote all the way up to the canonical SetCC type.
    982   EVT SVT = TLI.getSetCCResultType(N->getOpcode() == ISD::SELECT ?
    983                                    OpTy.getScalarType() : OpTy);
    984   Cond = PromoteTargetBoolean(Cond, SVT);
    985 
    986   return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
    987                                         N->getOperand(2)), 0);
    988 }
    989 
    990 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
    991   assert(OpNo == 0 && "Don't know how to promote this operand!");
    992 
    993   SDValue LHS = N->getOperand(0);
    994   SDValue RHS = N->getOperand(1);
    995   PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
    996 
    997   // The CC (#4) and the possible return values (#2 and #3) have legal types.
    998   return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
    999                                 N->getOperand(3), N->getOperand(4)), 0);
   1000 }
   1001 
   1002 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
   1003   assert(OpNo == 0 && "Don't know how to promote this operand!");
   1004 
   1005   SDValue LHS = N->getOperand(0);
   1006   SDValue RHS = N->getOperand(1);
   1007   PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
   1008 
   1009   // The CC (#2) is always legal.
   1010   return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
   1011 }
   1012 
   1013 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
   1014   return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
   1015                                 ZExtPromotedInteger(N->getOperand(1))), 0);
   1016 }
   1017 
   1018 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
   1019   SDValue Op = GetPromotedInteger(N->getOperand(0));
   1020   DebugLoc dl = N->getDebugLoc();
   1021   Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
   1022   return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
   1023                      Op, DAG.getValueType(N->getOperand(0).getValueType()));
   1024 }
   1025 
   1026 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
   1027   return SDValue(DAG.UpdateNodeOperands(N,
   1028                                 SExtPromotedInteger(N->getOperand(0))), 0);
   1029 }
   1030 
   1031 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
   1032   assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
   1033   SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
   1034   unsigned Alignment = N->getAlignment();
   1035   bool isVolatile = N->isVolatile();
   1036   bool isNonTemporal = N->isNonTemporal();
   1037   DebugLoc dl = N->getDebugLoc();
   1038 
   1039   SDValue Val = GetPromotedInteger(N->getValue());  // Get promoted value.
   1040 
   1041   // Truncate the value and store the result.
   1042   return DAG.getTruncStore(Ch, dl, Val, Ptr, N->getPointerInfo(),
   1043                            N->getMemoryVT(),
   1044                            isVolatile, isNonTemporal, Alignment);
   1045 }
   1046 
   1047 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
   1048   SDValue Op = GetPromotedInteger(N->getOperand(0));
   1049   return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), N->getValueType(0), Op);
   1050 }
   1051 
   1052 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
   1053   return SDValue(DAG.UpdateNodeOperands(N,
   1054                                 ZExtPromotedInteger(N->getOperand(0))), 0);
   1055 }
   1056 
   1057 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
   1058   DebugLoc dl = N->getDebugLoc();
   1059   SDValue Op = GetPromotedInteger(N->getOperand(0));
   1060   Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
   1061   return DAG.getZeroExtendInReg(Op, dl,
   1062                                 N->getOperand(0).getValueType().getScalarType());
   1063 }
   1064 
   1065 
   1066 //===----------------------------------------------------------------------===//
   1067 //  Integer Result Expansion
   1068 //===----------------------------------------------------------------------===//
   1069 
   1070 /// ExpandIntegerResult - This method is called when the specified result of the
   1071 /// specified node is found to need expansion.  At this point, the node may also
   1072 /// have invalid operands or may have other results that need promotion, we just
   1073 /// know that (at least) one result needs expansion.
   1074 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
   1075   DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); dbgs() << "\n");
   1076   SDValue Lo, Hi;
   1077   Lo = Hi = SDValue();
   1078 
   1079   // See if the target wants to custom expand this node.
   1080   if (CustomLowerNode(N, N->getValueType(ResNo), true))
   1081     return;
   1082 
   1083   switch (N->getOpcode()) {
   1084   default:
   1085 #ifndef NDEBUG
   1086     dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
   1087     N->dump(&DAG); dbgs() << "\n";
   1088 #endif
   1089     llvm_unreachable("Do not know how to expand the result of this operator!");
   1090 
   1091   case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
   1092   case ISD::SELECT:       SplitRes_SELECT(N, Lo, Hi); break;
   1093   case ISD::SELECT_CC:    SplitRes_SELECT_CC(N, Lo, Hi); break;
   1094   case ISD::UNDEF:        SplitRes_UNDEF(N, Lo, Hi); break;
   1095 
   1096   case ISD::BITCAST:            ExpandRes_BITCAST(N, Lo, Hi); break;
   1097   case ISD::BUILD_PAIR:         ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
   1098   case ISD::EXTRACT_ELEMENT:    ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
   1099   case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
   1100   case ISD::VAARG:              ExpandRes_VAARG(N, Lo, Hi); break;
   1101 
   1102   case ISD::ANY_EXTEND:  ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
   1103   case ISD::AssertSext:  ExpandIntRes_AssertSext(N, Lo, Hi); break;
   1104   case ISD::AssertZext:  ExpandIntRes_AssertZext(N, Lo, Hi); break;
   1105   case ISD::BSWAP:       ExpandIntRes_BSWAP(N, Lo, Hi); break;
   1106   case ISD::Constant:    ExpandIntRes_Constant(N, Lo, Hi); break;
   1107   case ISD::CTLZ_ZERO_UNDEF:
   1108   case ISD::CTLZ:        ExpandIntRes_CTLZ(N, Lo, Hi); break;
   1109   case ISD::CTPOP:       ExpandIntRes_CTPOP(N, Lo, Hi); break;
   1110   case ISD::CTTZ_ZERO_UNDEF:
   1111   case ISD::CTTZ:        ExpandIntRes_CTTZ(N, Lo, Hi); break;
   1112   case ISD::FP_TO_SINT:  ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
   1113   case ISD::FP_TO_UINT:  ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
   1114   case ISD::LOAD:        ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
   1115   case ISD::MUL:         ExpandIntRes_MUL(N, Lo, Hi); break;
   1116   case ISD::SDIV:        ExpandIntRes_SDIV(N, Lo, Hi); break;
   1117   case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
   1118   case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
   1119   case ISD::SREM:        ExpandIntRes_SREM(N, Lo, Hi); break;
   1120   case ISD::TRUNCATE:    ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
   1121   case ISD::UDIV:        ExpandIntRes_UDIV(N, Lo, Hi); break;
   1122   case ISD::UREM:        ExpandIntRes_UREM(N, Lo, Hi); break;
   1123   case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
   1124   case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
   1125 
   1126   case ISD::ATOMIC_LOAD_ADD:
   1127   case ISD::ATOMIC_LOAD_SUB:
   1128   case ISD::ATOMIC_LOAD_AND:
   1129   case ISD::ATOMIC_LOAD_OR:
   1130   case ISD::ATOMIC_LOAD_XOR:
   1131   case ISD::ATOMIC_LOAD_NAND:
   1132   case ISD::ATOMIC_LOAD_MIN:
   1133   case ISD::ATOMIC_LOAD_MAX:
   1134   case ISD::ATOMIC_LOAD_UMIN:
   1135   case ISD::ATOMIC_LOAD_UMAX:
   1136   case ISD::ATOMIC_SWAP: {
   1137     std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
   1138     SplitInteger(Tmp.first, Lo, Hi);
   1139     ReplaceValueWith(SDValue(N, 1), Tmp.second);
   1140     break;
   1141   }
   1142 
   1143   case ISD::AND:
   1144   case ISD::OR:
   1145   case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
   1146 
   1147   case ISD::ADD:
   1148   case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
   1149 
   1150   case ISD::ADDC:
   1151   case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
   1152 
   1153   case ISD::ADDE:
   1154   case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
   1155 
   1156   case ISD::SHL:
   1157   case ISD::SRA:
   1158   case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
   1159 
   1160   case ISD::SADDO:
   1161   case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
   1162   case ISD::UADDO:
   1163   case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
   1164   case ISD::UMULO:
   1165   case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
   1166   }
   1167 
   1168   // If Lo/Hi is null, the sub-method took care of registering results etc.
   1169   if (Lo.getNode())
   1170     SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
   1171 }
   1172 
   1173 /// Lower an atomic node to the appropriate builtin call.
   1174 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
   1175   unsigned Opc = Node->getOpcode();
   1176   MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
   1177   RTLIB::Libcall LC;
   1178 
   1179   switch (Opc) {
   1180   default:
   1181     llvm_unreachable("Unhandled atomic intrinsic Expand!");
   1182   case ISD::ATOMIC_SWAP:
   1183     switch (VT.SimpleTy) {
   1184     default: llvm_unreachable("Unexpected value type for atomic!");
   1185     case MVT::i8:  LC = RTLIB::SYNC_LOCK_TEST_AND_SET_1; break;
   1186     case MVT::i16: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_2; break;
   1187     case MVT::i32: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_4; break;
   1188     case MVT::i64: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_8; break;
   1189     }
   1190     break;
   1191   case ISD::ATOMIC_CMP_SWAP:
   1192     switch (VT.SimpleTy) {
   1193     default: llvm_unreachable("Unexpected value type for atomic!");
   1194     case MVT::i8:  LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_1; break;
   1195     case MVT::i16: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_2; break;
   1196     case MVT::i32: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_4; break;
   1197     case MVT::i64: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_8; break;
   1198     }
   1199     break;
   1200   case ISD::ATOMIC_LOAD_ADD:
   1201     switch (VT.SimpleTy) {
   1202     default: llvm_unreachable("Unexpected value type for atomic!");
   1203     case MVT::i8:  LC = RTLIB::SYNC_FETCH_AND_ADD_1; break;
   1204     case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_ADD_2; break;
   1205     case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_ADD_4; break;
   1206     case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_ADD_8; break;
   1207     }
   1208     break;
   1209   case ISD::ATOMIC_LOAD_SUB:
   1210     switch (VT.SimpleTy) {
   1211     default: llvm_unreachable("Unexpected value type for atomic!");
   1212     case MVT::i8:  LC = RTLIB::SYNC_FETCH_AND_SUB_1; break;
   1213     case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_SUB_2; break;
   1214     case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_SUB_4; break;
   1215     case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_SUB_8; break;
   1216     }
   1217     break;
   1218   case ISD::ATOMIC_LOAD_AND:
   1219     switch (VT.SimpleTy) {
   1220     default: llvm_unreachable("Unexpected value type for atomic!");
   1221     case MVT::i8:  LC = RTLIB::SYNC_FETCH_AND_AND_1; break;
   1222     case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_AND_2; break;
   1223     case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_AND_4; break;
   1224     case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_AND_8; break;
   1225     }
   1226     break;
   1227   case ISD::ATOMIC_LOAD_OR:
   1228     switch (VT.SimpleTy) {
   1229     default: llvm_unreachable("Unexpected value type for atomic!");
   1230     case MVT::i8:  LC = RTLIB::SYNC_FETCH_AND_OR_1; break;
   1231     case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_OR_2; break;
   1232     case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_OR_4; break;
   1233     case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_OR_8; break;
   1234     }
   1235     break;
   1236   case ISD::ATOMIC_LOAD_XOR:
   1237     switch (VT.SimpleTy) {
   1238     default: llvm_unreachable("Unexpected value type for atomic!");
   1239     case MVT::i8:  LC = RTLIB::SYNC_FETCH_AND_XOR_1; break;
   1240     case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_XOR_2; break;
   1241     case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_XOR_4; break;
   1242     case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_XOR_8; break;
   1243     }
   1244     break;
   1245   case ISD::ATOMIC_LOAD_NAND:
   1246     switch (VT.SimpleTy) {
   1247     default: llvm_unreachable("Unexpected value type for atomic!");
   1248     case MVT::i8:  LC = RTLIB::SYNC_FETCH_AND_NAND_1; break;
   1249     case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_NAND_2; break;
   1250     case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_NAND_4; break;
   1251     case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_NAND_8; break;
   1252     }
   1253     break;
   1254   }
   1255 
   1256   return ExpandChainLibCall(LC, Node, false);
   1257 }
   1258 
   1259 /// ExpandShiftByConstant - N is a shift by a value that needs to be expanded,
   1260 /// and the shift amount is a constant 'Amt'.  Expand the operation.
   1261 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, unsigned Amt,
   1262                                              SDValue &Lo, SDValue &Hi) {
   1263   DebugLoc DL = N->getDebugLoc();
   1264   // Expand the incoming operand to be shifted, so that we have its parts
   1265   SDValue InL, InH;
   1266   GetExpandedInteger(N->getOperand(0), InL, InH);
   1267 
   1268   EVT NVT = InL.getValueType();
   1269   unsigned VTBits = N->getValueType(0).getSizeInBits();
   1270   unsigned NVTBits = NVT.getSizeInBits();
   1271   EVT ShTy = N->getOperand(1).getValueType();
   1272 
   1273   if (N->getOpcode() == ISD::SHL) {
   1274     if (Amt > VTBits) {
   1275       Lo = Hi = DAG.getConstant(0, NVT);
   1276     } else if (Amt > NVTBits) {
   1277       Lo = DAG.getConstant(0, NVT);
   1278       Hi = DAG.getNode(ISD::SHL, DL,
   1279                        NVT, InL, DAG.getConstant(Amt-NVTBits, ShTy));
   1280     } else if (Amt == NVTBits) {
   1281       Lo = DAG.getConstant(0, NVT);
   1282       Hi = InL;
   1283     } else if (Amt == 1 &&
   1284                TLI.isOperationLegalOrCustom(ISD::ADDC,
   1285                               TLI.getTypeToExpandTo(*DAG.getContext(), NVT))) {
   1286       // Emit this X << 1 as X+X.
   1287       SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
   1288       SDValue LoOps[2] = { InL, InL };
   1289       Lo = DAG.getNode(ISD::ADDC, DL, VTList, LoOps, 2);
   1290       SDValue HiOps[3] = { InH, InH, Lo.getValue(1) };
   1291       Hi = DAG.getNode(ISD::ADDE, DL, VTList, HiOps, 3);
   1292     } else {
   1293       Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, ShTy));
   1294       Hi = DAG.getNode(ISD::OR, DL, NVT,
   1295                        DAG.getNode(ISD::SHL, DL, NVT, InH,
   1296                                    DAG.getConstant(Amt, ShTy)),
   1297                        DAG.getNode(ISD::SRL, DL, NVT, InL,
   1298                                    DAG.getConstant(NVTBits-Amt, ShTy)));
   1299     }
   1300     return;
   1301   }
   1302 
   1303   if (N->getOpcode() == ISD::SRL) {
   1304     if (Amt > VTBits) {
   1305       Lo = DAG.getConstant(0, NVT);
   1306       Hi = DAG.getConstant(0, NVT);
   1307     } else if (Amt > NVTBits) {
   1308       Lo = DAG.getNode(ISD::SRL, DL,
   1309                        NVT, InH, DAG.getConstant(Amt-NVTBits,ShTy));
   1310       Hi = DAG.getConstant(0, NVT);
   1311     } else if (Amt == NVTBits) {
   1312       Lo = InH;
   1313       Hi = DAG.getConstant(0, NVT);
   1314     } else {
   1315       Lo = DAG.getNode(ISD::OR, DL, NVT,
   1316                        DAG.getNode(ISD::SRL, DL, NVT, InL,
   1317                                    DAG.getConstant(Amt, ShTy)),
   1318                        DAG.getNode(ISD::SHL, DL, NVT, InH,
   1319                                    DAG.getConstant(NVTBits-Amt, ShTy)));
   1320       Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, ShTy));
   1321     }
   1322     return;
   1323   }
   1324 
   1325   assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
   1326   if (Amt > VTBits) {
   1327     Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
   1328                           DAG.getConstant(NVTBits-1, ShTy));
   1329   } else if (Amt > NVTBits) {
   1330     Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
   1331                      DAG.getConstant(Amt-NVTBits, ShTy));
   1332     Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
   1333                      DAG.getConstant(NVTBits-1, ShTy));
   1334   } else if (Amt == NVTBits) {
   1335     Lo = InH;
   1336     Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
   1337                      DAG.getConstant(NVTBits-1, ShTy));
   1338   } else {
   1339     Lo = DAG.getNode(ISD::OR, DL, NVT,
   1340                      DAG.getNode(ISD::SRL, DL, NVT, InL,
   1341                                  DAG.getConstant(Amt, ShTy)),
   1342                      DAG.getNode(ISD::SHL, DL, NVT, InH,
   1343                                  DAG.getConstant(NVTBits-Amt, ShTy)));
   1344     Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, ShTy));
   1345   }
   1346 }
   1347 
   1348 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
   1349 /// this shift based on knowledge of the high bit of the shift amount.  If we
   1350 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
   1351 /// shift amount.
   1352 bool DAGTypeLegalizer::
   1353 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
   1354   SDValue Amt = N->getOperand(1);
   1355   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
   1356   EVT ShTy = Amt.getValueType();
   1357   unsigned ShBits = ShTy.getScalarType().getSizeInBits();
   1358   unsigned NVTBits = NVT.getScalarType().getSizeInBits();
   1359   assert(isPowerOf2_32(NVTBits) &&
   1360          "Expanded integer type size not a power of two!");
   1361   DebugLoc dl = N->getDebugLoc();
   1362 
   1363   APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
   1364   APInt KnownZero, KnownOne;
   1365   DAG.ComputeMaskedBits(N->getOperand(1), KnownZero, KnownOne);
   1366 
   1367   // If we don't know anything about the high bits, exit.
   1368   if (((KnownZero|KnownOne) & HighBitMask) == 0)
   1369     return false;
   1370 
   1371   // Get the incoming operand to be shifted.
   1372   SDValue InL, InH;
   1373   GetExpandedInteger(N->getOperand(0), InL, InH);
   1374 
   1375   // If we know that any of the high bits of the shift amount are one, then we
   1376   // can do this as a couple of simple shifts.
   1377   if (KnownOne.intersects(HighBitMask)) {
   1378     // Mask out the high bit, which we know is set.
   1379     Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
   1380                       DAG.getConstant(~HighBitMask, ShTy));
   1381 
   1382     switch (N->getOpcode()) {
   1383     default: llvm_unreachable("Unknown shift");
   1384     case ISD::SHL:
   1385       Lo = DAG.getConstant(0, NVT);              // Low part is zero.
   1386       Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
   1387       return true;
   1388     case ISD::SRL:
   1389       Hi = DAG.getConstant(0, NVT);              // Hi part is zero.
   1390       Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
   1391       return true;
   1392     case ISD::SRA:
   1393       Hi = DAG.getNode(ISD::SRA, dl, NVT, InH,       // Sign extend high part.
   1394                        DAG.getConstant(NVTBits-1, ShTy));
   1395       Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
   1396       return true;
   1397     }
   1398   }
   1399 
   1400   // If we know that all of the high bits of the shift amount are zero, then we
   1401   // can do this as a couple of simple shifts.
   1402   if ((KnownZero & HighBitMask) == HighBitMask) {
   1403     // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
   1404     // shift if x is zero.  We can use XOR here because x is known to be smaller
   1405     // than 32.
   1406     SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
   1407                                DAG.getConstant(NVTBits-1, ShTy));
   1408 
   1409     unsigned Op1, Op2;
   1410     switch (N->getOpcode()) {
   1411     default: llvm_unreachable("Unknown shift");
   1412     case ISD::SHL:  Op1 = ISD::SHL; Op2 = ISD::SRL; break;
   1413     case ISD::SRL:
   1414     case ISD::SRA:  Op1 = ISD::SRL; Op2 = ISD::SHL; break;
   1415     }
   1416 
   1417     // When shifting right the arithmetic for Lo and Hi is swapped.
   1418     if (N->getOpcode() != ISD::SHL)
   1419       std::swap(InL, InH);
   1420 
   1421     // Use a little trick to get the bits that move from Lo to Hi. First
   1422     // shift by one bit.
   1423     SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, ShTy));
   1424     // Then compute the remaining shift with amount-1.
   1425     SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
   1426 
   1427     Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
   1428     Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
   1429 
   1430     if (N->getOpcode() != ISD::SHL)
   1431       std::swap(Hi, Lo);
   1432     return true;
   1433   }
   1434 
   1435   return false;
   1436 }
   1437 
   1438 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
   1439 /// of any size.
   1440 bool DAGTypeLegalizer::
   1441 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
   1442   SDValue Amt = N->getOperand(1);
   1443   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
   1444   EVT ShTy = Amt.getValueType();
   1445   unsigned NVTBits = NVT.getSizeInBits();
   1446   assert(isPowerOf2_32(NVTBits) &&
   1447          "Expanded integer type size not a power of two!");
   1448   DebugLoc dl = N->getDebugLoc();
   1449 
   1450   // Get the incoming operand to be shifted.
   1451   SDValue InL, InH;
   1452   GetExpandedInteger(N->getOperand(0), InL, InH);
   1453 
   1454   SDValue NVBitsNode = DAG.getConstant(NVTBits, ShTy);
   1455   SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
   1456   SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
   1457   SDValue isShort = DAG.getSetCC(dl, TLI.getSetCCResultType(ShTy),
   1458                                  Amt, NVBitsNode, ISD::SETULT);
   1459 
   1460   SDValue LoS, HiS, LoL, HiL;
   1461   switch (N->getOpcode()) {
   1462   default: llvm_unreachable("Unknown shift");
   1463   case ISD::SHL:
   1464     // Short: ShAmt < NVTBits
   1465     LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
   1466     HiS = DAG.getNode(ISD::OR, dl, NVT,
   1467                       DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
   1468     // FIXME: If Amt is zero, the following shift generates an undefined result
   1469     // on some architectures.
   1470                       DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
   1471 
   1472     // Long: ShAmt >= NVTBits
   1473     LoL = DAG.getConstant(0, NVT);                        // Lo part is zero.
   1474     HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
   1475 
   1476     Lo = DAG.getNode(ISD::SELECT, dl, NVT, isShort, LoS, LoL);
   1477     Hi = DAG.getNode(ISD::SELECT, dl, NVT, isShort, HiS, HiL);
   1478     return true;
   1479   case ISD::SRL:
   1480     // Short: ShAmt < NVTBits
   1481     HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
   1482     LoS = DAG.getNode(ISD::OR, dl, NVT,
   1483                       DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
   1484     // FIXME: If Amt is zero, the following shift generates an undefined result
   1485     // on some architectures.
   1486                       DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
   1487 
   1488     // Long: ShAmt >= NVTBits
   1489     HiL = DAG.getConstant(0, NVT);                        // Hi part is zero.
   1490     LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
   1491 
   1492     Lo = DAG.getNode(ISD::SELECT, dl, NVT, isShort, LoS, LoL);
   1493     Hi = DAG.getNode(ISD::SELECT, dl, NVT, isShort, HiS, HiL);
   1494     return true;
   1495   case ISD::SRA:
   1496     // Short: ShAmt < NVTBits
   1497     HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
   1498     LoS = DAG.getNode(ISD::OR, dl, NVT,
   1499                       DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
   1500     // FIXME: If Amt is zero, the following shift generates an undefined result
   1501     // on some architectures.
   1502                       DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
   1503 
   1504     // Long: ShAmt >= NVTBits
   1505     HiL = DAG.getNode(ISD::SRA, dl, NVT, InH,             // Sign of Hi part.
   1506                       DAG.getConstant(NVTBits-1, ShTy));
   1507     LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
   1508 
   1509     Lo = DAG.getNode(ISD::SELECT, dl, NVT, isShort, LoS, LoL);
   1510     Hi = DAG.getNode(ISD::SELECT, dl, NVT, isShort, HiS, HiL);
   1511     return true;
   1512   }
   1513 }
   1514 
   1515 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
   1516                                            SDValue &Lo, SDValue &Hi) {
   1517   DebugLoc dl = N->getDebugLoc();
   1518   // Expand the subcomponents.
   1519   SDValue LHSL, LHSH, RHSL, RHSH;
   1520   GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
   1521   GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
   1522 
   1523   EVT NVT = LHSL.getValueType();
   1524   SDValue LoOps[2] = { LHSL, RHSL };
   1525   SDValue HiOps[3] = { LHSH, RHSH };
   1526 
   1527   // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
   1528   // them.  TODO: Teach operation legalization how to expand unsupported
   1529   // ADDC/ADDE/SUBC/SUBE.  The problem is that these operations generate
   1530   // a carry of type MVT::Glue, but there doesn't seem to be any way to
   1531   // generate a value of this type in the expanded code sequence.
   1532   bool hasCarry =
   1533     TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
   1534                                    ISD::ADDC : ISD::SUBC,
   1535                                  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
   1536 
   1537   if (hasCarry) {
   1538     SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
   1539     if (N->getOpcode() == ISD::ADD) {
   1540       Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2);
   1541       HiOps[2] = Lo.getValue(1);
   1542       Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3);
   1543     } else {
   1544       Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps, 2);
   1545       HiOps[2] = Lo.getValue(1);
   1546       Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps, 3);
   1547     }
   1548     return;
   1549   }
   1550 
   1551   if (N->getOpcode() == ISD::ADD) {
   1552     Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps, 2);
   1553     Hi = DAG.getNode(ISD::ADD, dl, NVT, HiOps, 2);
   1554     SDValue Cmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Lo, LoOps[0],
   1555                                 ISD::SETULT);
   1556     SDValue Carry1 = DAG.getNode(ISD::SELECT, dl, NVT, Cmp1,
   1557                                  DAG.getConstant(1, NVT),
   1558                                  DAG.getConstant(0, NVT));
   1559     SDValue Cmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Lo, LoOps[1],
   1560                                 ISD::SETULT);
   1561     SDValue Carry2 = DAG.getNode(ISD::SELECT, dl, NVT, Cmp2,
   1562                                  DAG.getConstant(1, NVT), Carry1);
   1563     Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
   1564   } else {
   1565     Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps, 2);
   1566     Hi = DAG.getNode(ISD::SUB, dl, NVT, HiOps, 2);
   1567     SDValue Cmp =
   1568       DAG.getSetCC(dl, TLI.getSetCCResultType(LoOps[0].getValueType()),
   1569                    LoOps[0], LoOps[1], ISD::SETULT);
   1570     SDValue Borrow = DAG.getNode(ISD::SELECT, dl, NVT, Cmp,
   1571                                  DAG.getConstant(1, NVT),
   1572                                  DAG.getConstant(0, NVT));
   1573     Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
   1574   }
   1575 }
   1576 
   1577 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
   1578                                             SDValue &Lo, SDValue &Hi) {
   1579   // Expand the subcomponents.
   1580   SDValue LHSL, LHSH, RHSL, RHSH;
   1581   DebugLoc dl = N->getDebugLoc();
   1582   GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
   1583   GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
   1584   SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
   1585   SDValue LoOps[2] = { LHSL, RHSL };
   1586   SDValue HiOps[3] = { LHSH, RHSH };
   1587 
   1588   if (N->getOpcode() == ISD::ADDC) {
   1589     Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2);
   1590     HiOps[2] = Lo.getValue(1);
   1591     Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3);
   1592   } else {
   1593     Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps, 2);
   1594     HiOps[2] = Lo.getValue(1);
   1595     Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps, 3);
   1596   }
   1597 
   1598   // Legalized the flag result - switch anything that used the old flag to
   1599   // use the new one.
   1600   ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
   1601 }
   1602 
   1603 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
   1604                                             SDValue &Lo, SDValue &Hi) {
   1605   // Expand the subcomponents.
   1606   SDValue LHSL, LHSH, RHSL, RHSH;
   1607   DebugLoc dl = N->getDebugLoc();
   1608   GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
   1609   GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
   1610   SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
   1611   SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
   1612   SDValue HiOps[3] = { LHSH, RHSH };
   1613 
   1614   Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps, 3);
   1615   HiOps[2] = Lo.getValue(1);
   1616   Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps, 3);
   1617 
   1618   // Legalized the flag result - switch anything that used the old flag to
   1619   // use the new one.
   1620   ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
   1621 }
   1622 
   1623 void DAGTypeLegalizer::ExpandIntRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
   1624                                                  SDValue &Lo, SDValue &Hi) {
   1625   SDValue Res = DisintegrateMERGE_VALUES(N, ResNo);
   1626   SplitInteger(Res, Lo, Hi);
   1627 }
   1628 
   1629 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
   1630                                                SDValue &Lo, SDValue &Hi) {
   1631   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
   1632   DebugLoc dl = N->getDebugLoc();
   1633   SDValue Op = N->getOperand(0);
   1634   if (Op.getValueType().bitsLE(NVT)) {
   1635     // The low part is any extension of the input (which degenerates to a copy).
   1636     Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
   1637     Hi = DAG.getUNDEF(NVT);   // The high part is undefined.
   1638   } else {
   1639     // For example, extension of an i48 to an i64.  The operand type necessarily
   1640     // promotes to the result type, so will end up being expanded too.
   1641     assert(getTypeAction(Op.getValueType()) ==
   1642            TargetLowering::TypePromoteInteger &&
   1643            "Only know how to promote this result!");
   1644     SDValue Res = GetPromotedInteger(Op);
   1645     assert(Res.getValueType() == N->getValueType(0) &&
   1646            "Operand over promoted?");
   1647     // Split the promoted operand.  This will simplify when it is expanded.
   1648     SplitInteger(Res, Lo, Hi);
   1649   }
   1650 }
   1651 
   1652 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
   1653                                                SDValue &Lo, SDValue &Hi) {
   1654   DebugLoc dl = N->getDebugLoc();
   1655   GetExpandedInteger(N->getOperand(0), Lo, Hi);
   1656   EVT NVT = Lo.getValueType();
   1657   EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
   1658   unsigned NVTBits = NVT.getSizeInBits();
   1659   unsigned EVTBits = EVT.getSizeInBits();
   1660 
   1661   if (NVTBits < EVTBits) {
   1662     Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
   1663                      DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
   1664                                                         EVTBits - NVTBits)));
   1665   } else {
   1666     Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
   1667     // The high part replicates the sign bit of Lo, make it explicit.
   1668     Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
   1669                      DAG.getConstant(NVTBits-1, TLI.getPointerTy()));
   1670   }
   1671 }
   1672 
   1673 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
   1674                                                SDValue &Lo, SDValue &Hi) {
   1675   DebugLoc dl = N->getDebugLoc();
   1676   GetExpandedInteger(N->getOperand(0), Lo, Hi);
   1677   EVT NVT = Lo.getValueType();
   1678   EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
   1679   unsigned NVTBits = NVT.getSizeInBits();
   1680   unsigned EVTBits = EVT.getSizeInBits();
   1681 
   1682   if (NVTBits < EVTBits) {
   1683     Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
   1684                      DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
   1685                                                         EVTBits - NVTBits)));
   1686   } else {
   1687     Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
   1688     // The high part must be zero, make it explicit.
   1689     Hi = DAG.getConstant(0, NVT);
   1690   }
   1691 }
   1692 
   1693 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
   1694                                           SDValue &Lo, SDValue &Hi) {
   1695   DebugLoc dl = N->getDebugLoc();
   1696   GetExpandedInteger(N->getOperand(0), Hi, Lo);  // Note swapped operands.
   1697   Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
   1698   Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
   1699 }
   1700 
   1701 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
   1702                                              SDValue &Lo, SDValue &Hi) {
   1703   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
   1704   unsigned NBitWidth = NVT.getSizeInBits();
   1705   const APInt &Cst = cast<ConstantSDNode>(N)->getAPIntValue();
   1706   Lo = DAG.getConstant(Cst.trunc(NBitWidth), NVT);
   1707   Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), NVT);
   1708 }
   1709 
   1710 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
   1711                                          SDValue &Lo, SDValue &Hi) {
   1712   DebugLoc dl = N->getDebugLoc();
   1713   // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
   1714   GetExpandedInteger(N->getOperand(0), Lo, Hi);
   1715   EVT NVT = Lo.getValueType();
   1716 
   1717   SDValue HiNotZero = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Hi,
   1718                                    DAG.getConstant(0, NVT), ISD::SETNE);
   1719 
   1720   SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
   1721   SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
   1722 
   1723   Lo = DAG.getNode(ISD::SELECT, dl, NVT, HiNotZero, HiLZ,
   1724                    DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
   1725                                DAG.getConstant(NVT.getSizeInBits(), NVT)));
   1726   Hi = DAG.getConstant(0, NVT);
   1727 }
   1728 
   1729 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
   1730                                           SDValue &Lo, SDValue &Hi) {
   1731   DebugLoc dl = N->getDebugLoc();
   1732   // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
   1733   GetExpandedInteger(N->getOperand(0), Lo, Hi);
   1734   EVT NVT = Lo.getValueType();
   1735   Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
   1736                    DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
   1737   Hi = DAG.getConstant(0, NVT);
   1738 }
   1739 
   1740 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
   1741                                          SDValue &Lo, SDValue &Hi) {
   1742   DebugLoc dl = N->getDebugLoc();
   1743   // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
   1744   GetExpandedInteger(N->getOperand(0), Lo, Hi);
   1745   EVT NVT = Lo.getValueType();
   1746 
   1747   SDValue LoNotZero = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Lo,
   1748                                    DAG.getConstant(0, NVT), ISD::SETNE);
   1749 
   1750   SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
   1751   SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
   1752 
   1753   Lo = DAG.getNode(ISD::SELECT, dl, NVT, LoNotZero, LoLZ,
   1754                    DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
   1755                                DAG.getConstant(NVT.getSizeInBits(), NVT)));
   1756   Hi = DAG.getConstant(0, NVT);
   1757 }
   1758 
   1759 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
   1760                                                SDValue &Hi) {
   1761   DebugLoc dl = N->getDebugLoc();
   1762   EVT VT = N->getValueType(0);
   1763   SDValue Op = N->getOperand(0);
   1764   RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT);
   1765   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
   1766   SplitInteger(MakeLibCall(LC, VT, &Op, 1, true/*irrelevant*/, dl), Lo, Hi);
   1767 }
   1768 
   1769 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
   1770                                                SDValue &Hi) {
   1771   DebugLoc dl = N->getDebugLoc();
   1772   EVT VT = N->getValueType(0);
   1773   SDValue Op = N->getOperand(0);
   1774   RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT);
   1775   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
   1776   SplitInteger(MakeLibCall(LC, VT, &Op, 1, false/*irrelevant*/, dl), Lo, Hi);
   1777 }
   1778 
   1779 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
   1780                                          SDValue &Lo, SDValue &Hi) {
   1781   if (ISD::isNormalLoad(N)) {
   1782     ExpandRes_NormalLoad(N, Lo, Hi);
   1783     return;
   1784   }
   1785 
   1786   assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
   1787 
   1788   EVT VT = N->getValueType(0);
   1789   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
   1790   SDValue Ch  = N->getChain();
   1791   SDValue Ptr = N->getBasePtr();
   1792   ISD::LoadExtType ExtType = N->getExtensionType();
   1793   unsigned Alignment = N->getAlignment();
   1794   bool isVolatile = N->isVolatile();
   1795   bool isNonTemporal = N->isNonTemporal();
   1796   bool isInvariant = N->isInvariant();
   1797   DebugLoc dl = N->getDebugLoc();
   1798 
   1799   assert(NVT.isByteSized() && "Expanded type not byte sized!");
   1800 
   1801   if (N->getMemoryVT().bitsLE(NVT)) {
   1802     EVT MemVT = N->getMemoryVT();
   1803 
   1804     Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
   1805                         MemVT, isVolatile, isNonTemporal, Alignment);
   1806 
   1807     // Remember the chain.
   1808     Ch = Lo.getValue(1);
   1809 
   1810     if (ExtType == ISD::SEXTLOAD) {
   1811       // The high part is obtained by SRA'ing all but one of the bits of the
   1812       // lo part.
   1813       unsigned LoSize = Lo.getValueType().getSizeInBits();
   1814       Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
   1815                        DAG.getConstant(LoSize-1, TLI.getPointerTy()));
   1816     } else if (ExtType == ISD::ZEXTLOAD) {
   1817       // The high part is just a zero.
   1818       Hi = DAG.getConstant(0, NVT);
   1819     } else {
   1820       assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
   1821       // The high part is undefined.
   1822       Hi = DAG.getUNDEF(NVT);
   1823     }
   1824   } else if (TLI.isLittleEndian()) {
   1825     // Little-endian - low bits are at low addresses.
   1826     Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(),
   1827                      isVolatile, isNonTemporal, isInvariant, Alignment);
   1828 
   1829     unsigned ExcessBits =
   1830       N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
   1831     EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
   1832 
   1833     // Increment the pointer to the other half.
   1834     unsigned IncrementSize = NVT.getSizeInBits()/8;
   1835     Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
   1836                       DAG.getIntPtrConstant(IncrementSize));
   1837     Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
   1838                         N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
   1839                         isVolatile, isNonTemporal,
   1840                         MinAlign(Alignment, IncrementSize));
   1841 
   1842     // Build a factor node to remember that this load is independent of the
   1843     // other one.
   1844     Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
   1845                      Hi.getValue(1));
   1846   } else {
   1847     // Big-endian - high bits are at low addresses.  Favor aligned loads at
   1848     // the cost of some bit-fiddling.
   1849     EVT MemVT = N->getMemoryVT();
   1850     unsigned EBytes = MemVT.getStoreSize();
   1851     unsigned IncrementSize = NVT.getSizeInBits()/8;
   1852     unsigned ExcessBits = (EBytes - IncrementSize)*8;
   1853 
   1854     // Load both the high bits and maybe some of the low bits.
   1855     Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
   1856                         EVT::getIntegerVT(*DAG.getContext(),
   1857                                           MemVT.getSizeInBits() - ExcessBits),
   1858                         isVolatile, isNonTemporal, Alignment);
   1859 
   1860     // Increment the pointer to the other half.
   1861     Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
   1862                       DAG.getIntPtrConstant(IncrementSize));
   1863     // Load the rest of the low bits.
   1864     Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
   1865                         N->getPointerInfo().getWithOffset(IncrementSize),
   1866                         EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
   1867                         isVolatile, isNonTemporal,
   1868                         MinAlign(Alignment, IncrementSize));
   1869 
   1870     // Build a factor node to remember that this load is independent of the
   1871     // other one.
   1872     Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
   1873                      Hi.getValue(1));
   1874 
   1875     if (ExcessBits < NVT.getSizeInBits()) {
   1876       // Transfer low bits from the bottom of Hi to the top of Lo.
   1877       Lo = DAG.getNode(ISD::OR, dl, NVT, Lo,
   1878                        DAG.getNode(ISD::SHL, dl, NVT, Hi,
   1879                                    DAG.getConstant(ExcessBits,
   1880                                                    TLI.getPointerTy())));
   1881       // Move high bits to the right position in Hi.
   1882       Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl,
   1883                        NVT, Hi,
   1884                        DAG.getConstant(NVT.getSizeInBits() - ExcessBits,
   1885                                        TLI.getPointerTy()));
   1886     }
   1887   }
   1888 
   1889   // Legalized the chain result - switch anything that used the old chain to
   1890   // use the new one.
   1891   ReplaceValueWith(SDValue(N, 1), Ch);
   1892 }
   1893 
   1894 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
   1895                                             SDValue &Lo, SDValue &Hi) {
   1896   DebugLoc dl = N->getDebugLoc();
   1897   SDValue LL, LH, RL, RH;
   1898   GetExpandedInteger(N->getOperand(0), LL, LH);
   1899   GetExpandedInteger(N->getOperand(1), RL, RH);
   1900   Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
   1901   Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
   1902 }
   1903 
   1904 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
   1905                                         SDValue &Lo, SDValue &Hi) {
   1906   EVT VT = N->getValueType(0);
   1907   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
   1908   DebugLoc dl = N->getDebugLoc();
   1909 
   1910   bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, NVT);
   1911   bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, NVT);
   1912   bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, NVT);
   1913   bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, NVT);
   1914   if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) {
   1915     SDValue LL, LH, RL, RH;
   1916     GetExpandedInteger(N->getOperand(0), LL, LH);
   1917     GetExpandedInteger(N->getOperand(1), RL, RH);
   1918     unsigned OuterBitSize = VT.getSizeInBits();
   1919     unsigned InnerBitSize = NVT.getSizeInBits();
   1920     unsigned LHSSB = DAG.ComputeNumSignBits(N->getOperand(0));
   1921     unsigned RHSSB = DAG.ComputeNumSignBits(N->getOperand(1));
   1922 
   1923     APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);
   1924     if (DAG.MaskedValueIsZero(N->getOperand(0), HighMask) &&
   1925         DAG.MaskedValueIsZero(N->getOperand(1), HighMask)) {
   1926       // The inputs are both zero-extended.
   1927       if (HasUMUL_LOHI) {
   1928         // We can emit a umul_lohi.
   1929         Lo = DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(NVT, NVT), LL, RL);
   1930         Hi = SDValue(Lo.getNode(), 1);
   1931         return;
   1932       }
   1933       if (HasMULHU) {
   1934         // We can emit a mulhu+mul.
   1935         Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL);
   1936         Hi = DAG.getNode(ISD::MULHU, dl, NVT, LL, RL);
   1937         return;
   1938       }
   1939     }
   1940     if (LHSSB > InnerBitSize && RHSSB > InnerBitSize) {
   1941       // The input values are both sign-extended.
   1942       if (HasSMUL_LOHI) {
   1943         // We can emit a smul_lohi.
   1944         Lo = DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(NVT, NVT), LL, RL);
   1945         Hi = SDValue(Lo.getNode(), 1);
   1946         return;
   1947       }
   1948       if (HasMULHS) {
   1949         // We can emit a mulhs+mul.
   1950         Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL);
   1951         Hi = DAG.getNode(ISD::MULHS, dl, NVT, LL, RL);
   1952         return;
   1953       }
   1954     }
   1955     if (HasUMUL_LOHI) {
   1956       // Lo,Hi = umul LHS, RHS.
   1957       SDValue UMulLOHI = DAG.getNode(ISD::UMUL_LOHI, dl,
   1958                                        DAG.getVTList(NVT, NVT), LL, RL);
   1959       Lo = UMulLOHI;
   1960       Hi = UMulLOHI.getValue(1);
   1961       RH = DAG.getNode(ISD::MUL, dl, NVT, LL, RH);
   1962       LH = DAG.getNode(ISD::MUL, dl, NVT, LH, RL);
   1963       Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, RH);
   1964       Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, LH);
   1965       return;
   1966     }
   1967     if (HasMULHU) {
   1968       Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL);
   1969       Hi = DAG.getNode(ISD::MULHU, dl, NVT, LL, RL);
   1970       RH = DAG.getNode(ISD::MUL, dl, NVT, LL, RH);
   1971       LH = DAG.getNode(ISD::MUL, dl, NVT, LH, RL);
   1972       Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, RH);
   1973       Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, LH);
   1974       return;
   1975     }
   1976   }
   1977 
   1978   // If nothing else, we can make a libcall.
   1979   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
   1980   if (VT == MVT::i16)
   1981     LC = RTLIB::MUL_I16;
   1982   else if (VT == MVT::i32)
   1983     LC = RTLIB::MUL_I32;
   1984   else if (VT == MVT::i64)
   1985     LC = RTLIB::MUL_I64;
   1986   else if (VT == MVT::i128)
   1987     LC = RTLIB::MUL_I128;
   1988   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported MUL!");
   1989 
   1990   SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
   1991   SplitInteger(MakeLibCall(LC, VT, Ops, 2, true/*irrelevant*/, dl), Lo, Hi);
   1992 }
   1993 
   1994 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
   1995                                              SDValue &Lo, SDValue &Hi) {
   1996   SDValue LHS = Node->getOperand(0);
   1997   SDValue RHS = Node->getOperand(1);
   1998   DebugLoc dl = Node->getDebugLoc();
   1999 
   2000   // Expand the result by simply replacing it with the equivalent
   2001   // non-overflow-checking operation.
   2002   SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
   2003                             ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
   2004                             LHS, RHS);
   2005   SplitInteger(Sum, Lo, Hi);
   2006 
   2007   // Compute the overflow.
   2008   //
   2009   //   LHSSign -> LHS >= 0
   2010   //   RHSSign -> RHS >= 0
   2011   //   SumSign -> Sum >= 0
   2012   //
   2013   //   Add:
   2014   //   Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
   2015   //   Sub:
   2016   //   Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
   2017   //
   2018   EVT OType = Node->getValueType(1);
   2019   SDValue Zero = DAG.getConstant(0, LHS.getValueType());
   2020 
   2021   SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
   2022   SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
   2023   SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
   2024                                     Node->getOpcode() == ISD::SADDO ?
   2025                                     ISD::SETEQ : ISD::SETNE);
   2026 
   2027   SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
   2028   SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
   2029 
   2030   SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
   2031 
   2032   // Use the calculated overflow everywhere.
   2033   ReplaceValueWith(SDValue(Node, 1), Cmp);
   2034 }
   2035 
   2036 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
   2037                                          SDValue &Lo, SDValue &Hi) {
   2038   EVT VT = N->getValueType(0);
   2039   DebugLoc dl = N->getDebugLoc();
   2040 
   2041   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
   2042   if (VT == MVT::i16)
   2043     LC = RTLIB::SDIV_I16;
   2044   else if (VT == MVT::i32)
   2045     LC = RTLIB::SDIV_I32;
   2046   else if (VT == MVT::i64)
   2047     LC = RTLIB::SDIV_I64;
   2048   else if (VT == MVT::i128)
   2049     LC = RTLIB::SDIV_I128;
   2050   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
   2051 
   2052   SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
   2053   SplitInteger(MakeLibCall(LC, VT, Ops, 2, true, dl), Lo, Hi);
   2054 }
   2055 
   2056 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
   2057                                           SDValue &Lo, SDValue &Hi) {
   2058   EVT VT = N->getValueType(0);
   2059   DebugLoc dl = N->getDebugLoc();
   2060 
   2061   // If we can emit an efficient shift operation, do so now.  Check to see if
   2062   // the RHS is a constant.
   2063   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
   2064     return ExpandShiftByConstant(N, CN->getZExtValue(), Lo, Hi);
   2065 
   2066   // If we can determine that the high bit of the shift is zero or one, even if
   2067   // the low bits are variable, emit this shift in an optimized form.
   2068   if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
   2069     return;
   2070 
   2071   // If this target supports shift_PARTS, use it.  First, map to the _PARTS opc.
   2072   unsigned PartsOpc;
   2073   if (N->getOpcode() == ISD::SHL) {
   2074     PartsOpc = ISD::SHL_PARTS;
   2075   } else if (N->getOpcode() == ISD::SRL) {
   2076     PartsOpc = ISD::SRL_PARTS;
   2077   } else {
   2078     assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
   2079     PartsOpc = ISD::SRA_PARTS;
   2080   }
   2081 
   2082   // Next check to see if the target supports this SHL_PARTS operation or if it
   2083   // will custom expand it.
   2084   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
   2085   TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
   2086   if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
   2087       Action == TargetLowering::Custom) {
   2088     // Expand the subcomponents.
   2089     SDValue LHSL, LHSH;
   2090     GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
   2091 
   2092     SDValue Ops[] = { LHSL, LHSH, N->getOperand(1) };
   2093     EVT VT = LHSL.getValueType();
   2094     Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops, 3);
   2095     Hi = Lo.getValue(1);
   2096     return;
   2097   }
   2098 
   2099   // Otherwise, emit a libcall.
   2100   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
   2101   bool isSigned;
   2102   if (N->getOpcode() == ISD::SHL) {
   2103     isSigned = false; /*sign irrelevant*/
   2104     if (VT == MVT::i16)
   2105       LC = RTLIB::SHL_I16;
   2106     else if (VT == MVT::i32)
   2107       LC = RTLIB::SHL_I32;
   2108     else if (VT == MVT::i64)
   2109       LC = RTLIB::SHL_I64;
   2110     else if (VT == MVT::i128)
   2111       LC = RTLIB::SHL_I128;
   2112   } else if (N->getOpcode() == ISD::SRL) {
   2113     isSigned = false;
   2114     if (VT == MVT::i16)
   2115       LC = RTLIB::SRL_I16;
   2116     else if (VT == MVT::i32)
   2117       LC = RTLIB::SRL_I32;
   2118     else if (VT == MVT::i64)
   2119       LC = RTLIB::SRL_I64;
   2120     else if (VT == MVT::i128)
   2121       LC = RTLIB::SRL_I128;
   2122   } else {
   2123     assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
   2124     isSigned = true;
   2125     if (VT == MVT::i16)
   2126       LC = RTLIB::SRA_I16;
   2127     else if (VT == MVT::i32)
   2128       LC = RTLIB::SRA_I32;
   2129     else if (VT == MVT::i64)
   2130       LC = RTLIB::SRA_I64;
   2131     else if (VT == MVT::i128)
   2132       LC = RTLIB::SRA_I128;
   2133   }
   2134 
   2135   if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
   2136     SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
   2137     SplitInteger(MakeLibCall(LC, VT, Ops, 2, isSigned, dl), Lo, Hi);
   2138     return;
   2139   }
   2140 
   2141   if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
   2142     llvm_unreachable("Unsupported shift!");
   2143 }
   2144 
   2145 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
   2146                                                 SDValue &Lo, SDValue &Hi) {
   2147   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
   2148   DebugLoc dl = N->getDebugLoc();
   2149   SDValue Op = N->getOperand(0);
   2150   if (Op.getValueType().bitsLE(NVT)) {
   2151     // The low part is sign extension of the input (degenerates to a copy).
   2152     Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
   2153     // The high part is obtained by SRA'ing all but one of the bits of low part.
   2154     unsigned LoSize = NVT.getSizeInBits();
   2155     Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
   2156                      DAG.getConstant(LoSize-1, TLI.getPointerTy()));
   2157   } else {
   2158     // For example, extension of an i48 to an i64.  The operand type necessarily
   2159     // promotes to the result type, so will end up being expanded too.
   2160     assert(getTypeAction(Op.getValueType()) ==
   2161            TargetLowering::TypePromoteInteger &&
   2162            "Only know how to promote this result!");
   2163     SDValue Res = GetPromotedInteger(Op);
   2164     assert(Res.getValueType() == N->getValueType(0) &&
   2165            "Operand over promoted?");
   2166     // Split the promoted operand.  This will simplify when it is expanded.
   2167     SplitInteger(Res, Lo, Hi);
   2168     unsigned ExcessBits =
   2169       Op.getValueType().getSizeInBits() - NVT.getSizeInBits();
   2170     Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
   2171                      DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
   2172                                                         ExcessBits)));
   2173   }
   2174 }
   2175 
   2176 void DAGTypeLegalizer::
   2177 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
   2178   DebugLoc dl = N->getDebugLoc();
   2179   GetExpandedInteger(N->getOperand(0), Lo, Hi);
   2180   EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
   2181 
   2182   if (EVT.bitsLE(Lo.getValueType())) {
   2183     // sext_inreg the low part if needed.
   2184     Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
   2185                      N->getOperand(1));
   2186 
   2187     // The high part gets the sign extension from the lo-part.  This handles
   2188     // things like sextinreg V:i64 from i8.
   2189     Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
   2190                      DAG.getConstant(Hi.getValueType().getSizeInBits()-1,
   2191                                      TLI.getPointerTy()));
   2192   } else {
   2193     // For example, extension of an i48 to an i64.  Leave the low part alone,
   2194     // sext_inreg the high part.
   2195     unsigned ExcessBits =
   2196       EVT.getSizeInBits() - Lo.getValueType().getSizeInBits();
   2197     Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
   2198                      DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
   2199                                                         ExcessBits)));
   2200   }
   2201 }
   2202 
   2203 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
   2204                                          SDValue &Lo, SDValue &Hi) {
   2205   EVT VT = N->getValueType(0);
   2206   DebugLoc dl = N->getDebugLoc();
   2207 
   2208   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
   2209   if (VT == MVT::i16)
   2210     LC = RTLIB::SREM_I16;
   2211   else if (VT == MVT::i32)
   2212     LC = RTLIB::SREM_I32;
   2213   else if (VT == MVT::i64)
   2214     LC = RTLIB::SREM_I64;
   2215   else if (VT == MVT::i128)
   2216     LC = RTLIB::SREM_I128;
   2217   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
   2218 
   2219   SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
   2220   SplitInteger(MakeLibCall(LC, VT, Ops, 2, true, dl), Lo, Hi);
   2221 }
   2222 
   2223 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
   2224                                              SDValue &Lo, SDValue &Hi) {
   2225   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
   2226   DebugLoc dl = N->getDebugLoc();
   2227   Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
   2228   Hi = DAG.getNode(ISD::SRL, dl,
   2229                    N->getOperand(0).getValueType(), N->getOperand(0),
   2230                    DAG.getConstant(NVT.getSizeInBits(), TLI.getPointerTy()));
   2231   Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
   2232 }
   2233 
   2234 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
   2235                                              SDValue &Lo, SDValue &Hi) {
   2236   SDValue LHS = N->getOperand(0);
   2237   SDValue RHS = N->getOperand(1);
   2238   DebugLoc dl = N->getDebugLoc();
   2239 
   2240   // Expand the result by simply replacing it with the equivalent
   2241   // non-overflow-checking operation.
   2242   SDValue Sum = DAG.getNode(N->getOpcode() == ISD::UADDO ?
   2243                             ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
   2244                             LHS, RHS);
   2245   SplitInteger(Sum, Lo, Hi);
   2246 
   2247   // Calculate the overflow: addition overflows iff a + b < a, and subtraction
   2248   // overflows iff a - b > a.
   2249   SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS,
   2250                              N->getOpcode () == ISD::UADDO ?
   2251                              ISD::SETULT : ISD::SETUGT);
   2252 
   2253   // Use the calculated overflow everywhere.
   2254   ReplaceValueWith(SDValue(N, 1), Ofl);
   2255 }
   2256 
   2257 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
   2258                                           SDValue &Lo, SDValue &Hi) {
   2259   EVT VT = N->getValueType(0);
   2260   Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
   2261   EVT PtrVT = TLI.getPointerTy();
   2262   Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
   2263   DebugLoc dl = N->getDebugLoc();
   2264 
   2265   // A divide for UMULO should be faster than a function call.
   2266   if (N->getOpcode() == ISD::UMULO) {
   2267     SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
   2268     DebugLoc DL = N->getDebugLoc();
   2269 
   2270     SDValue MUL = DAG.getNode(ISD::MUL, DL, LHS.getValueType(), LHS, RHS);
   2271     SplitInteger(MUL, Lo, Hi);
   2272 
   2273     // A divide for UMULO will be faster than a function call. Select to
   2274     // make sure we aren't using 0.
   2275     SDValue isZero = DAG.getSetCC(dl, TLI.getSetCCResultType(VT),
   2276 				  RHS, DAG.getConstant(0, VT), ISD::SETNE);
   2277     SDValue NotZero = DAG.getNode(ISD::SELECT, dl, VT, isZero,
   2278 				  DAG.getConstant(1, VT), RHS);
   2279     SDValue DIV = DAG.getNode(ISD::UDIV, DL, LHS.getValueType(), MUL, NotZero);
   2280     SDValue Overflow;
   2281     Overflow = DAG.getSetCC(DL, N->getValueType(1), DIV, LHS, ISD::SETNE);
   2282     ReplaceValueWith(SDValue(N, 1), Overflow);
   2283     return;
   2284   }
   2285 
   2286   // Replace this with a libcall that will check overflow.
   2287   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
   2288   if (VT == MVT::i32)
   2289     LC = RTLIB::MULO_I32;
   2290   else if (VT == MVT::i64)
   2291     LC = RTLIB::MULO_I64;
   2292   else if (VT == MVT::i128)
   2293     LC = RTLIB::MULO_I128;
   2294   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
   2295 
   2296   SDValue Temp = DAG.CreateStackTemporary(PtrVT);
   2297   // Temporary for the overflow value, default it to zero.
   2298   SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl,
   2299 			       DAG.getConstant(0, PtrVT), Temp,
   2300 			       MachinePointerInfo(), false, false, 0);
   2301 
   2302   TargetLowering::ArgListTy Args;
   2303   TargetLowering::ArgListEntry Entry;
   2304   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
   2305     EVT ArgVT = N->getOperand(i).getValueType();
   2306     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
   2307     Entry.Node = N->getOperand(i);
   2308     Entry.Ty = ArgTy;
   2309     Entry.isSExt = true;
   2310     Entry.isZExt = false;
   2311     Args.push_back(Entry);
   2312   }
   2313 
   2314   // Also pass the address of the overflow check.
   2315   Entry.Node = Temp;
   2316   Entry.Ty = PtrTy->getPointerTo();
   2317   Entry.isSExt = true;
   2318   Entry.isZExt = false;
   2319   Args.push_back(Entry);
   2320 
   2321   SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
   2322   std::pair<SDValue, SDValue> CallInfo =
   2323     TLI.LowerCallTo(Chain, RetTy, true, false, false, false,
   2324 		    0, TLI.getLibcallCallingConv(LC),
   2325                     /*isTailCall=*/false,
   2326 		    /*doesNotReturn=*/false, /*isReturnValueUsed=*/true,
   2327                     Func, Args, DAG, dl);
   2328 
   2329   SplitInteger(CallInfo.first, Lo, Hi);
   2330   SDValue Temp2 = DAG.getLoad(PtrVT, dl, CallInfo.second, Temp,
   2331 			      MachinePointerInfo(), false, false, false, 0);
   2332   SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
   2333                              DAG.getConstant(0, PtrVT),
   2334                              ISD::SETNE);
   2335   // Use the overflow from the libcall everywhere.
   2336   ReplaceValueWith(SDValue(N, 1), Ofl);
   2337 }
   2338 
   2339 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
   2340                                          SDValue &Lo, SDValue &Hi) {
   2341   EVT VT = N->getValueType(0);
   2342   DebugLoc dl = N->getDebugLoc();
   2343 
   2344   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
   2345   if (VT == MVT::i16)
   2346     LC = RTLIB::UDIV_I16;
   2347   else if (VT == MVT::i32)
   2348     LC = RTLIB::UDIV_I32;
   2349   else if (VT == MVT::i64)
   2350     LC = RTLIB::UDIV_I64;
   2351   else if (VT == MVT::i128)
   2352     LC = RTLIB::UDIV_I128;
   2353   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
   2354 
   2355   SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
   2356   SplitInteger(MakeLibCall(LC, VT, Ops, 2, false, dl), Lo, Hi);
   2357 }
   2358 
   2359 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
   2360                                          SDValue &Lo, SDValue &Hi) {
   2361   EVT VT = N->getValueType(0);
   2362   DebugLoc dl = N->getDebugLoc();
   2363 
   2364   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
   2365   if (VT == MVT::i16)
   2366     LC = RTLIB::UREM_I16;
   2367   else if (VT == MVT::i32)
   2368     LC = RTLIB::UREM_I32;
   2369   else if (VT == MVT::i64)
   2370     LC = RTLIB::UREM_I64;
   2371   else if (VT == MVT::i128)
   2372     LC = RTLIB::UREM_I128;
   2373   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
   2374 
   2375   SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
   2376   SplitInteger(MakeLibCall(LC, VT, Ops, 2, false, dl), Lo, Hi);
   2377 }
   2378 
   2379 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
   2380                                                 SDValue &Lo, SDValue &Hi) {
   2381   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
   2382   DebugLoc dl = N->getDebugLoc();
   2383   SDValue Op = N->getOperand(0);
   2384   if (Op.getValueType().bitsLE(NVT)) {
   2385     // The low part is zero extension of the input (degenerates to a copy).
   2386     Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
   2387     Hi = DAG.getConstant(0, NVT);   // The high part is just a zero.
   2388   } else {
   2389     // For example, extension of an i48 to an i64.  The operand type necessarily
   2390     // promotes to the result type, so will end up being expanded too.
   2391     assert(getTypeAction(Op.getValueType()) ==
   2392            TargetLowering::TypePromoteInteger &&
   2393            "Only know how to promote this result!");
   2394     SDValue Res = GetPromotedInteger(Op);
   2395     assert(Res.getValueType() == N->getValueType(0) &&
   2396            "Operand over promoted?");
   2397     // Split the promoted operand.  This will simplify when it is expanded.
   2398     SplitInteger(Res, Lo, Hi);
   2399     unsigned ExcessBits =
   2400       Op.getValueType().getSizeInBits() - NVT.getSizeInBits();
   2401     Hi = DAG.getZeroExtendInReg(Hi, dl,
   2402                                 EVT::getIntegerVT(*DAG.getContext(),
   2403                                                   ExcessBits));
   2404   }
   2405 }
   2406 
   2407 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
   2408                                                 SDValue &Lo, SDValue &Hi) {
   2409   DebugLoc dl = N->getDebugLoc();
   2410   EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
   2411   SDValue Zero = DAG.getConstant(0, VT);
   2412   SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, VT,
   2413                                N->getOperand(0),
   2414                                N->getOperand(1), Zero, Zero,
   2415                                cast<AtomicSDNode>(N)->getMemOperand(),
   2416                                cast<AtomicSDNode>(N)->getOrdering(),
   2417                                cast<AtomicSDNode>(N)->getSynchScope());
   2418   ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
   2419   ReplaceValueWith(SDValue(N, 1), Swap.getValue(1));
   2420 }
   2421 
   2422 //===----------------------------------------------------------------------===//
   2423 //  Integer Operand Expansion
   2424 //===----------------------------------------------------------------------===//
   2425 
   2426 /// ExpandIntegerOperand - This method is called when the specified operand of
   2427 /// the specified node is found to need expansion.  At this point, all of the
   2428 /// result types of the node are known to be legal, but other operands of the
   2429 /// node may need promotion or expansion as well as the specified one.
   2430 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
   2431   DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); dbgs() << "\n");
   2432   SDValue Res = SDValue();
   2433 
   2434   if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
   2435     return false;
   2436 
   2437   switch (N->getOpcode()) {
   2438   default:
   2439   #ifndef NDEBUG
   2440     dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
   2441     N->dump(&DAG); dbgs() << "\n";
   2442   #endif
   2443     llvm_unreachable("Do not know how to expand this operator's operand!");
   2444 
   2445   case ISD::BITCAST:           Res = ExpandOp_BITCAST(N); break;
   2446   case ISD::BR_CC:             Res = ExpandIntOp_BR_CC(N); break;
   2447   case ISD::BUILD_VECTOR:      Res = ExpandOp_BUILD_VECTOR(N); break;
   2448   case ISD::EXTRACT_ELEMENT:   Res = ExpandOp_EXTRACT_ELEMENT(N); break;
   2449   case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
   2450   case ISD::SCALAR_TO_VECTOR:  Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
   2451   case ISD::SELECT_CC:         Res = ExpandIntOp_SELECT_CC(N); break;
   2452   case ISD::SETCC:             Res = ExpandIntOp_SETCC(N); break;
   2453   case ISD::SINT_TO_FP:        Res = ExpandIntOp_SINT_TO_FP(N); break;
   2454   case ISD::STORE:   Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
   2455   case ISD::TRUNCATE:          Res = ExpandIntOp_TRUNCATE(N); break;
   2456   case ISD::UINT_TO_FP:        Res = ExpandIntOp_UINT_TO_FP(N); break;
   2457 
   2458   case ISD::SHL:
   2459   case ISD::SRA:
   2460   case ISD::SRL:
   2461   case ISD::ROTL:
   2462   case ISD::ROTR:              Res = ExpandIntOp_Shift(N); break;
   2463   case ISD::RETURNADDR:
   2464   case ISD::FRAMEADDR:         Res = ExpandIntOp_RETURNADDR(N); break;
   2465 
   2466   case ISD::ATOMIC_STORE:      Res = ExpandIntOp_ATOMIC_STORE(N); break;
   2467   }
   2468 
   2469   // If the result is null, the sub-method took care of registering results etc.
   2470   if (!Res.getNode()) return false;
   2471 
   2472   // If the result is N, the sub-method updated N in place.  Tell the legalizer
   2473   // core about this.
   2474   if (Res.getNode() == N)
   2475     return true;
   2476 
   2477   assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
   2478          "Invalid operand expansion");
   2479 
   2480   ReplaceValueWith(SDValue(N, 0), Res);
   2481   return false;
   2482 }
   2483 
   2484 /// IntegerExpandSetCCOperands - Expand the operands of a comparison.  This code
   2485 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
   2486 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
   2487                                                   SDValue &NewRHS,
   2488                                                   ISD::CondCode &CCCode,
   2489                                                   DebugLoc dl) {
   2490   SDValue LHSLo, LHSHi, RHSLo, RHSHi;
   2491   GetExpandedInteger(NewLHS, LHSLo, LHSHi);
   2492   GetExpandedInteger(NewRHS, RHSLo, RHSHi);
   2493 
   2494   if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
   2495     if (RHSLo == RHSHi) {
   2496       if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
   2497         if (RHSCST->isAllOnesValue()) {
   2498           // Equality comparison to -1.
   2499           NewLHS = DAG.getNode(ISD::AND, dl,
   2500                                LHSLo.getValueType(), LHSLo, LHSHi);
   2501           NewRHS = RHSLo;
   2502           return;
   2503         }
   2504       }
   2505     }
   2506 
   2507     NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
   2508     NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
   2509     NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
   2510     NewRHS = DAG.getConstant(0, NewLHS.getValueType());
   2511     return;
   2512   }
   2513 
   2514   // If this is a comparison of the sign bit, just look at the top part.
   2515   // X > -1,  x < 0
   2516   if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
   2517     if ((CCCode == ISD::SETLT && CST->isNullValue()) ||     // X < 0
   2518         (CCCode == ISD::SETGT && CST->isAllOnesValue())) {  // X > -1
   2519       NewLHS = LHSHi;
   2520       NewRHS = RHSHi;
   2521       return;
   2522     }
   2523 
   2524   // FIXME: This generated code sucks.
   2525   ISD::CondCode LowCC;
   2526   switch (CCCode) {
   2527   default: llvm_unreachable("Unknown integer setcc!");
   2528   case ISD::SETLT:
   2529   case ISD::SETULT: LowCC = ISD::SETULT; break;
   2530   case ISD::SETGT:
   2531   case ISD::SETUGT: LowCC = ISD::SETUGT; break;
   2532   case ISD::SETLE:
   2533   case ISD::SETULE: LowCC = ISD::SETULE; break;
   2534   case ISD::SETGE:
   2535   case ISD::SETUGE: LowCC = ISD::SETUGE; break;
   2536   }
   2537 
   2538   // Tmp1 = lo(op1) < lo(op2)   // Always unsigned comparison
   2539   // Tmp2 = hi(op1) < hi(op2)   // Signedness depends on operands
   2540   // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2;
   2541 
   2542   // NOTE: on targets without efficient SELECT of bools, we can always use
   2543   // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
   2544   TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, true, NULL);
   2545   SDValue Tmp1, Tmp2;
   2546   Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSLo.getValueType()),
   2547                            LHSLo, RHSLo, LowCC, false, DagCombineInfo, dl);
   2548   if (!Tmp1.getNode())
   2549     Tmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSLo.getValueType()),
   2550                         LHSLo, RHSLo, LowCC);
   2551   Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()),
   2552                            LHSHi, RHSHi, CCCode, false, DagCombineInfo, dl);
   2553   if (!Tmp2.getNode())
   2554     Tmp2 = DAG.getNode(ISD::SETCC, dl,
   2555                        TLI.getSetCCResultType(LHSHi.getValueType()),
   2556                        LHSHi, RHSHi, DAG.getCondCode(CCCode));
   2557 
   2558   ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.getNode());
   2559   ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.getNode());
   2560   if ((Tmp1C && Tmp1C->isNullValue()) ||
   2561       (Tmp2C && Tmp2C->isNullValue() &&
   2562        (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
   2563         CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) ||
   2564       (Tmp2C && Tmp2C->getAPIntValue() == 1 &&
   2565        (CCCode == ISD::SETLT || CCCode == ISD::SETGT ||
   2566         CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) {
   2567     // low part is known false, returns high part.
   2568     // For LE / GE, if high part is known false, ignore the low part.
   2569     // For LT / GT, if high part is known true, ignore the low part.
   2570     NewLHS = Tmp2;
   2571     NewRHS = SDValue();
   2572     return;
   2573   }
   2574 
   2575   NewLHS = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()),
   2576                              LHSHi, RHSHi, ISD::SETEQ, false,
   2577                              DagCombineInfo, dl);
   2578   if (!NewLHS.getNode())
   2579     NewLHS = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSHi.getValueType()),
   2580                           LHSHi, RHSHi, ISD::SETEQ);
   2581   NewLHS = DAG.getNode(ISD::SELECT, dl, Tmp1.getValueType(),
   2582                        NewLHS, Tmp1, Tmp2);
   2583   NewRHS = SDValue();
   2584 }
   2585 
   2586 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
   2587   SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
   2588   ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
   2589   IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, N->getDebugLoc());
   2590 
   2591   // If ExpandSetCCOperands returned a scalar, we need to compare the result
   2592   // against zero to select between true and false values.
   2593   if (NewRHS.getNode() == 0) {
   2594     NewRHS = DAG.getConstant(0, NewLHS.getValueType());
   2595     CCCode = ISD::SETNE;
   2596   }
   2597 
   2598   // Update N to have the operands specified.
   2599   return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
   2600                                 DAG.getCondCode(CCCode), NewLHS, NewRHS,
   2601                                 N->getOperand(4)), 0);
   2602 }
   2603 
   2604 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
   2605   SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
   2606   ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
   2607   IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, N->getDebugLoc());
   2608 
   2609   // If ExpandSetCCOperands returned a scalar, we need to compare the result
   2610   // against zero to select between true and false values.
   2611   if (NewRHS.getNode() == 0) {
   2612     NewRHS = DAG.getConstant(0, NewLHS.getValueType());
   2613     CCCode = ISD::SETNE;
   2614   }
   2615 
   2616   // Update N to have the operands specified.
   2617   return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
   2618                                 N->getOperand(2), N->getOperand(3),
   2619                                 DAG.getCondCode(CCCode)), 0);
   2620 }
   2621 
   2622 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
   2623   SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
   2624   ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
   2625   IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, N->getDebugLoc());
   2626 
   2627   // If ExpandSetCCOperands returned a scalar, use it.
   2628   if (NewRHS.getNode() == 0) {
   2629     assert(NewLHS.getValueType() == N->getValueType(0) &&
   2630            "Unexpected setcc expansion!");
   2631     return NewLHS;
   2632   }
   2633 
   2634   // Otherwise, update N to have the operands specified.
   2635   return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
   2636                                 DAG.getCondCode(CCCode)), 0);
   2637 }
   2638 
   2639 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
   2640   // The value being shifted is legal, but the shift amount is too big.
   2641   // It follows that either the result of the shift is undefined, or the
   2642   // upper half of the shift amount is zero.  Just use the lower half.
   2643   SDValue Lo, Hi;
   2644   GetExpandedInteger(N->getOperand(1), Lo, Hi);
   2645   return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
   2646 }
   2647 
   2648 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
   2649   // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant.  This
   2650   // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
   2651   // constant to valid type.
   2652   SDValue Lo, Hi;
   2653   GetExpandedInteger(N->getOperand(0), Lo, Hi);
   2654   return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
   2655 }
   2656 
   2657 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
   2658   SDValue Op = N->getOperand(0);
   2659   EVT DstVT = N->getValueType(0);
   2660   RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT);
   2661   assert(LC != RTLIB::UNKNOWN_LIBCALL &&
   2662          "Don't know how to expand this SINT_TO_FP!");
   2663   return MakeLibCall(LC, DstVT, &Op, 1, true, N->getDebugLoc());
   2664 }
   2665 
   2666 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
   2667   if (ISD::isNormalStore(N))
   2668     return ExpandOp_NormalStore(N, OpNo);
   2669 
   2670   assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
   2671   assert(OpNo == 1 && "Can only expand the stored value so far");
   2672 
   2673   EVT VT = N->getOperand(1).getValueType();
   2674   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
   2675   SDValue Ch  = N->getChain();
   2676   SDValue Ptr = N->getBasePtr();
   2677   unsigned Alignment = N->getAlignment();
   2678   bool isVolatile = N->isVolatile();
   2679   bool isNonTemporal = N->isNonTemporal();
   2680   DebugLoc dl = N->getDebugLoc();
   2681   SDValue Lo, Hi;
   2682 
   2683   assert(NVT.isByteSized() && "Expanded type not byte sized!");
   2684 
   2685   if (N->getMemoryVT().bitsLE(NVT)) {
   2686     GetExpandedInteger(N->getValue(), Lo, Hi);
   2687     return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
   2688                              N->getMemoryVT(), isVolatile, isNonTemporal,
   2689                              Alignment);
   2690   }
   2691 
   2692   if (TLI.isLittleEndian()) {
   2693     // Little-endian - low bits are at low addresses.
   2694     GetExpandedInteger(N->getValue(), Lo, Hi);
   2695 
   2696     Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
   2697                       isVolatile, isNonTemporal, Alignment);
   2698 
   2699     unsigned ExcessBits =
   2700       N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
   2701     EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
   2702 
   2703     // Increment the pointer to the other half.
   2704     unsigned IncrementSize = NVT.getSizeInBits()/8;
   2705     Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
   2706                       DAG.getIntPtrConstant(IncrementSize));
   2707     Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr,
   2708                            N->getPointerInfo().getWithOffset(IncrementSize),
   2709                            NEVT, isVolatile, isNonTemporal,
   2710                            MinAlign(Alignment, IncrementSize));
   2711     return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
   2712   }
   2713 
   2714   // Big-endian - high bits are at low addresses.  Favor aligned stores at
   2715   // the cost of some bit-fiddling.
   2716   GetExpandedInteger(N->getValue(), Lo, Hi);
   2717 
   2718   EVT ExtVT = N->getMemoryVT();
   2719   unsigned EBytes = ExtVT.getStoreSize();
   2720   unsigned IncrementSize = NVT.getSizeInBits()/8;
   2721   unsigned ExcessBits = (EBytes - IncrementSize)*8;
   2722   EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
   2723                                ExtVT.getSizeInBits() - ExcessBits);
   2724 
   2725   if (ExcessBits < NVT.getSizeInBits()) {
   2726     // Transfer high bits from the top of Lo to the bottom of Hi.
   2727     Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
   2728                      DAG.getConstant(NVT.getSizeInBits() - ExcessBits,
   2729                                      TLI.getPointerTy()));
   2730     Hi = DAG.getNode(ISD::OR, dl, NVT, Hi,
   2731                      DAG.getNode(ISD::SRL, dl, NVT, Lo,
   2732                                  DAG.getConstant(ExcessBits,
   2733                                                  TLI.getPointerTy())));
   2734   }
   2735 
   2736   // Store both the high bits and maybe some of the low bits.
   2737   Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(),
   2738                          HiVT, isVolatile, isNonTemporal, Alignment);
   2739 
   2740   // Increment the pointer to the other half.
   2741   Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
   2742                     DAG.getIntPtrConstant(IncrementSize));
   2743   // Store the lowest ExcessBits bits in the second half.
   2744   Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
   2745                          N->getPointerInfo().getWithOffset(IncrementSize),
   2746                          EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
   2747                          isVolatile, isNonTemporal,
   2748                          MinAlign(Alignment, IncrementSize));
   2749   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
   2750 }
   2751 
   2752 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
   2753   SDValue InL, InH;
   2754   GetExpandedInteger(N->getOperand(0), InL, InH);
   2755   // Just truncate the low part of the source.
   2756   return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), N->getValueType(0), InL);
   2757 }
   2758 
   2759 static const fltSemantics *EVTToAPFloatSemantics(EVT VT) {
   2760   switch (VT.getSimpleVT().SimpleTy) {
   2761   default: llvm_unreachable("Unknown FP format");
   2762   case MVT::f32:     return &APFloat::IEEEsingle;
   2763   case MVT::f64:     return &APFloat::IEEEdouble;
   2764   case MVT::f80:     return &APFloat::x87DoubleExtended;
   2765   case MVT::f128:    return &APFloat::IEEEquad;
   2766   case MVT::ppcf128: return &APFloat::PPCDoubleDouble;
   2767   }
   2768 }
   2769 
   2770 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
   2771   SDValue Op = N->getOperand(0);
   2772   EVT SrcVT = Op.getValueType();
   2773   EVT DstVT = N->getValueType(0);
   2774   DebugLoc dl = N->getDebugLoc();
   2775 
   2776   // The following optimization is valid only if every value in SrcVT (when
   2777   // treated as signed) is representable in DstVT.  Check that the mantissa
   2778   // size of DstVT is >= than the number of bits in SrcVT -1.
   2779   const fltSemantics *sem = EVTToAPFloatSemantics(DstVT);
   2780   if (APFloat::semanticsPrecision(*sem) >= SrcVT.getSizeInBits()-1 &&
   2781       TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom){
   2782     // Do a signed conversion then adjust the result.
   2783     SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
   2784     SignedConv = TLI.LowerOperation(SignedConv, DAG);
   2785 
   2786     // The result of the signed conversion needs adjusting if the 'sign bit' of
   2787     // the incoming integer was set.  To handle this, we dynamically test to see
   2788     // if it is set, and, if so, add a fudge factor.
   2789 
   2790     const uint64_t F32TwoE32  = 0x4F800000ULL;
   2791     const uint64_t F32TwoE64  = 0x5F800000ULL;
   2792     const uint64_t F32TwoE128 = 0x7F800000ULL;
   2793 
   2794     APInt FF(32, 0);
   2795     if (SrcVT == MVT::i32)
   2796       FF = APInt(32, F32TwoE32);
   2797     else if (SrcVT == MVT::i64)
   2798       FF = APInt(32, F32TwoE64);
   2799     else if (SrcVT == MVT::i128)
   2800       FF = APInt(32, F32TwoE128);
   2801     else
   2802       llvm_unreachable("Unsupported UINT_TO_FP!");
   2803 
   2804     // Check whether the sign bit is set.
   2805     SDValue Lo, Hi;
   2806     GetExpandedInteger(Op, Lo, Hi);
   2807     SDValue SignSet = DAG.getSetCC(dl,
   2808                                    TLI.getSetCCResultType(Hi.getValueType()),
   2809                                    Hi, DAG.getConstant(0, Hi.getValueType()),
   2810                                    ISD::SETLT);
   2811 
   2812     // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
   2813     SDValue FudgePtr = DAG.getConstantPool(
   2814                                ConstantInt::get(*DAG.getContext(), FF.zext(64)),
   2815                                            TLI.getPointerTy());
   2816 
   2817     // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
   2818     SDValue Zero = DAG.getIntPtrConstant(0);
   2819     SDValue Four = DAG.getIntPtrConstant(4);
   2820     if (TLI.isBigEndian()) std::swap(Zero, Four);
   2821     SDValue Offset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), SignSet,
   2822                                  Zero, Four);
   2823     unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
   2824     FudgePtr = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), FudgePtr, Offset);
   2825     Alignment = std::min(Alignment, 4u);
   2826 
   2827     // Load the value out, extending it from f32 to the destination float type.
   2828     // FIXME: Avoid the extend by constructing the right constant pool?
   2829     SDValue Fudge = DAG.getExtLoad(ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(),
   2830                                    FudgePtr,
   2831                                    MachinePointerInfo::getConstantPool(),
   2832                                    MVT::f32,
   2833                                    false, false, Alignment);
   2834     return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
   2835   }
   2836 
   2837   // Otherwise, use a libcall.
   2838   RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
   2839   assert(LC != RTLIB::UNKNOWN_LIBCALL &&
   2840          "Don't know how to expand this UINT_TO_FP!");
   2841   return MakeLibCall(LC, DstVT, &Op, 1, true, dl);
   2842 }
   2843 
   2844 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
   2845   DebugLoc dl = N->getDebugLoc();
   2846   SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
   2847                                cast<AtomicSDNode>(N)->getMemoryVT(),
   2848                                N->getOperand(0),
   2849                                N->getOperand(1), N->getOperand(2),
   2850                                cast<AtomicSDNode>(N)->getMemOperand(),
   2851                                cast<AtomicSDNode>(N)->getOrdering(),
   2852                                cast<AtomicSDNode>(N)->getSynchScope());
   2853   return Swap.getValue(1);
   2854 }
   2855 
   2856 
   2857 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
   2858   SDValue InOp0 = N->getOperand(0);
   2859   EVT InVT = InOp0.getValueType();
   2860 
   2861   EVT OutVT = N->getValueType(0);
   2862   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
   2863   assert(NOutVT.isVector() && "This type must be promoted to a vector type");
   2864   unsigned OutNumElems = OutVT.getVectorNumElements();
   2865   EVT NOutVTElem = NOutVT.getVectorElementType();
   2866 
   2867   DebugLoc dl = N->getDebugLoc();
   2868   SDValue BaseIdx = N->getOperand(1);
   2869 
   2870   SmallVector<SDValue, 8> Ops;
   2871   Ops.reserve(OutNumElems);
   2872   for (unsigned i = 0; i != OutNumElems; ++i) {
   2873 
   2874     // Extract the element from the original vector.
   2875     SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
   2876       BaseIdx, DAG.getIntPtrConstant(i));
   2877     SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
   2878       InVT.getVectorElementType(), N->getOperand(0), Index);
   2879 
   2880     SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext);
   2881     // Insert the converted element to the new vector.
   2882     Ops.push_back(Op);
   2883   }
   2884 
   2885   return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, &Ops[0], Ops.size());
   2886 }
   2887 
   2888 
   2889 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
   2890   ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
   2891   EVT VT = N->getValueType(0);
   2892   DebugLoc dl = N->getDebugLoc();
   2893 
   2894   unsigned NumElts = VT.getVectorNumElements();
   2895   SmallVector<int, 8> NewMask;
   2896   for (unsigned i = 0; i != NumElts; ++i) {
   2897     NewMask.push_back(SV->getMaskElt(i));
   2898   }
   2899 
   2900   SDValue V0 = GetPromotedInteger(N->getOperand(0));
   2901   SDValue V1 = GetPromotedInteger(N->getOperand(1));
   2902   EVT OutVT = V0.getValueType();
   2903 
   2904   return DAG.getVectorShuffle(OutVT, dl, V0, V1, &NewMask[0]);
   2905 }
   2906 
   2907 
   2908 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
   2909   EVT OutVT = N->getValueType(0);
   2910   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
   2911   assert(NOutVT.isVector() && "This type must be promoted to a vector type");
   2912   unsigned NumElems = N->getNumOperands();
   2913   EVT NOutVTElem = NOutVT.getVectorElementType();
   2914 
   2915   DebugLoc dl = N->getDebugLoc();
   2916 
   2917   SmallVector<SDValue, 8> Ops;
   2918   Ops.reserve(NumElems);
   2919   for (unsigned i = 0; i != NumElems; ++i) {
   2920     SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
   2921     Ops.push_back(Op);
   2922   }
   2923 
   2924   return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, &Ops[0], Ops.size());
   2925 }
   2926 
   2927 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
   2928 
   2929   DebugLoc dl = N->getDebugLoc();
   2930 
   2931   assert(!N->getOperand(0).getValueType().isVector() &&
   2932          "Input must be a scalar");
   2933 
   2934   EVT OutVT = N->getValueType(0);
   2935   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
   2936   assert(NOutVT.isVector() && "This type must be promoted to a vector type");
   2937   EVT NOutVTElem = NOutVT.getVectorElementType();
   2938 
   2939   SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
   2940 
   2941   return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
   2942 }
   2943 
   2944 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
   2945   DebugLoc dl = N->getDebugLoc();
   2946 
   2947   EVT OutVT = N->getValueType(0);
   2948   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
   2949   assert(NOutVT.isVector() && "This type must be promoted to a vector type");
   2950 
   2951   EVT InElemTy = OutVT.getVectorElementType();
   2952   EVT OutElemTy = NOutVT.getVectorElementType();
   2953 
   2954   unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
   2955   unsigned NumOutElem = NOutVT.getVectorNumElements();
   2956   unsigned NumOperands = N->getNumOperands();
   2957   assert(NumElem * NumOperands == NumOutElem &&
   2958          "Unexpected number of elements");
   2959 
   2960   // Take the elements from the first vector.
   2961   SmallVector<SDValue, 8> Ops(NumOutElem);
   2962   for (unsigned i = 0; i < NumOperands; ++i) {
   2963     SDValue Op = N->getOperand(i);
   2964     for (unsigned j = 0; j < NumElem; ++j) {
   2965       SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
   2966                                 InElemTy, Op, DAG.getIntPtrConstant(j));
   2967       Ops[i * NumElem + j] = DAG.getNode(ISD::ANY_EXTEND, dl, OutElemTy, Ext);
   2968     }
   2969   }
   2970 
   2971   return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, &Ops[0], Ops.size());
   2972 }
   2973 
   2974 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
   2975   EVT OutVT = N->getValueType(0);
   2976   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
   2977   assert(NOutVT.isVector() && "This type must be promoted to a vector type");
   2978 
   2979   EVT NOutVTElem = NOutVT.getVectorElementType();
   2980 
   2981   DebugLoc dl = N->getDebugLoc();
   2982   SDValue V0 = GetPromotedInteger(N->getOperand(0));
   2983 
   2984   SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
   2985     NOutVTElem, N->getOperand(1));
   2986   return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
   2987     V0, ConvElem, N->getOperand(2));
   2988 }
   2989 
   2990 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
   2991   DebugLoc dl = N->getDebugLoc();
   2992   SDValue V0 = GetPromotedInteger(N->getOperand(0));
   2993   SDValue V1 = N->getOperand(1);
   2994   SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
   2995     V0->getValueType(0).getScalarType(), V0, V1);
   2996 
   2997   // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
   2998   // element types. If this is the case then we need to expand the outgoing
   2999   // value and not truncate it.
   3000   return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
   3001 }
   3002 
   3003 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
   3004   DebugLoc dl = N->getDebugLoc();
   3005   unsigned NumElems = N->getNumOperands();
   3006 
   3007   EVT RetSclrTy = N->getValueType(0).getVectorElementType();
   3008 
   3009   SmallVector<SDValue, 8> NewOps;
   3010   NewOps.reserve(NumElems);
   3011 
   3012   // For each incoming vector
   3013   for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
   3014     SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
   3015     EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
   3016     unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
   3017 
   3018     for (unsigned i=0; i<NumElem; ++i) {
   3019       // Extract element from incoming vector
   3020       SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy,
   3021       Incoming, DAG.getIntPtrConstant(i));
   3022       SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
   3023       NewOps.push_back(Tr);
   3024     }
   3025   }
   3026 
   3027   return DAG.getNode(ISD::BUILD_VECTOR, dl,  N->getValueType(0),
   3028     &NewOps[0], NewOps.size());
   3029   }
   3030