Lines Matching refs:ISD
266 TLI.isLoadExtLegal(ISD::EXTLOAD, SVT) &&
279 DAG.getExtLoad(ISD::EXTLOAD, dl, OrigVT,
296 assert(ST->getAddressingMode() == ISD::UNINDEXED &&
311 SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val);
352 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
354 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
364 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Store, StackPtr,
376 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Stores[0],
393 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);
400 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
409 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
418 assert(LD->getAddressingMode() == ISD::UNINDEXED &&
434 SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad);
436 Result = DAG.getNode(ISD::FP_EXTEND, dl, VT, Result);
471 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
472 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
479 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr,
492 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Stores[0],
516 ISD::LoadExtType HiExtType = LD->getExtensionType();
519 if (HiExtType == ISD::NON_EXTLOAD)
520 HiExtType = ISD::ZEXTLOAD;
525 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(),
528 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
538 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
540 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
549 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);
550 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo);
552 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
590 unsigned CastOpc = IdxVT.bitsGT(PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND;
594 Tmp3 = DAG.getNode(ISD::MUL, dl, IdxVT, Tmp3,DAG.getConstant(EltSize, IdxVT));
595 SDValue StackPtr2 = DAG.getNode(ISD::ADD, dl, IdxVT, Tmp3, StackPtr);
615 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
678 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2,
684 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
694 if (Node->getOpcode() == ISD::TargetConstant) // Allow illegal target nodes.
708 Node->getOperand(i).getOpcode() == ISD::TargetConstant) &&
718 case ISD::INTRINSIC_W_CHAIN:
719 case ISD::INTRINSIC_WO_CHAIN:
720 case ISD::INTRINSIC_VOID:
721 case ISD::STACKSAVE:
724 case ISD::VAARG:
730 case ISD::SINT_TO_FP:
731 case ISD::UINT_TO_FP:
732 case ISD::EXTRACT_VECTOR_ELT:
736 case ISD::FP_ROUND_INREG:
737 case ISD::SIGN_EXTEND_INREG: {
742 case ISD::ATOMIC_STORE: {
747 case ISD::SELECT_CC:
748 case ISD::SETCC:
749 case ISD::BR_CC: {
750 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
751 Node->getOpcode() == ISD::SETCC ? 2 : 1;
752 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
754 ISD::CondCode CCCode =
758 ISD::SELECT_CC)
766 case ISD::LOAD:
767 case ISD::STORE:
772 case ISD::CALLSEQ_START:
773 case ISD::CALLSEQ_END:
779 case ISD::EXTRACT_ELEMENT:
780 case ISD::FLT_ROUNDS_:
781 case ISD::SADDO:
782 case ISD::SSUBO:
783 case ISD::UADDO:
784 case ISD::USUBO:
785 case ISD::SMULO:
786 case ISD::UMULO:
787 case ISD::FPOWI:
788 case ISD::MERGE_VALUES:
789 case ISD::EH_RETURN:
790 case ISD::FRAME_TO_ARGS_OFFSET:
791 case ISD::EH_SJLJ_SETJMP:
792 case ISD::EH_SJLJ_LONGJMP:
799 case ISD::INIT_TRAMPOLINE:
800 case ISD::ADJUST_TRAMPOLINE:
801 case ISD::FRAMEADDR:
802 case ISD::RETURNADDR:
810 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
824 case ISD::SHL:
825 case ISD::SRL:
826 case ISD::SRA:
827 case ISD::ROTL:
828 case ISD::ROTR:
838 case ISD::SRL_PARTS:
839 case ISD::SRA_PARTS:
840 case ISD::SHL_PARTS:
903 case ISD::CALLSEQ_START:
904 case ISD::CALLSEQ_END:
906 case ISD::LOAD: {
911 ISD::LoadExtType ExtType = LD->getExtensionType();
912 if (ExtType == ISD::NON_EXTLOAD) {
947 Tmp3 = DAG.getNode(ISD::BITCAST, dl, VT, Tmp1);
986 ISD::LoadExtType NewExtType =
987 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
996 if (ExtType == ISD::SEXTLOAD)
998 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
1001 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
1003 Result = DAG.getNode(ISD::AssertZext, dl,
1026 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0),
1033 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2,
1042 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1046 Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi,
1051 Tmp1 = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
1062 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2,
1064 Lo = DAG.getExtLoad(ISD::ZEXTLOAD,
1072 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1076 Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi,
1081 Tmp1 = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
1117 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, SrcVT) && TLI.isTypeLegal(SrcVT)) {
1124 case ISD::EXTLOAD:
1126 ISD::FP_EXTEND : ISD::ANY_EXTEND);
1128 case ISD::SEXTLOAD: ExtendOp = ISD::SIGN_EXTEND; break;
1129 case ISD::ZEXTLOAD: ExtendOp = ISD::ZERO_EXTEND; break;
1144 assert(ExtType != ISD::EXTLOAD &&
1148 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, Node->getValueType(0),
1153 if (ExtType == ISD::SEXTLOAD)
1154 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
1175 case ISD::STORE: {
1192 switch (TLI.getOperationAction(ISD::STORE, VT)) {
1212 Tmp3 = DAG.getNode(ISD::BITCAST, dl,
1213 TLI.getTypeToPromoteTo(ISD::STORE, VT), Tmp3);
1264 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2,
1266 Hi = DAG.getNode(ISD::SRL, dl, Tmp3.getValueType(), Tmp3,
1277 Hi = DAG.getNode(ISD::SRL, dl, Tmp3.getValueType(), Tmp3,
1285 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2,
1294 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
1319 Tmp3 = DAG.getNode(ISD::TRUNCATE, dl, StVT, Tmp3);
1345 Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx,
1349 Idx = DAG.getNode(ISD::TRUNCATE, dl, TLI.getPointerTy(), Idx);
1351 Idx = DAG.getNode(ISD::ZERO_EXTEND, dl, TLI.getPointerTy(), Idx);
1353 StackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, StackPtr);
1358 return DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1388 Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx,
1392 Idx = DAG.getNode(ISD::TRUNCATE, dl, TLI.getPointerTy(), Idx);
1394 Idx = DAG.getNode(ISD::ZERO_EXTEND, dl, TLI.getPointerTy(), Idx);
1396 SDValue SubStackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx,
1426 if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue;
1431 Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
1449 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1471 SignBit = DAG.getNode(ISD::BITCAST, dl, IVT, Tmp2);
1492 LoadPtr = DAG.getNode(ISD::ADD, dl, LoadPtr.getValueType(),
1502 SignBit = DAG.getNode(ISD::SHL, dl, LoadTy, SignBit,
1510 ISD::SETLT);
1512 SDValue AbsVal = DAG.getNode(ISD::FABS, dl, Tmp1.getValueType(), Tmp1);
1515 return DAG.getNode(ISD::SELECT, dl, AbsVal.getValueType(), SignBit,
1516 DAG.getNode(ISD::FNEG, dl, AbsVal.getValueType(), AbsVal),
1542 SP = DAG.getNode(ISD::AND, dl, VT, SP,
1544 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
1562 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1569 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
1573 case ISD::SETOEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETO; Opc = ISD::AND; break;
1574 case ISD::SETOGT: CC1 = ISD::SETGT; CC2 = ISD::SETO; Opc = ISD::AND; break;
1575 case ISD::SETOGE: CC1 = ISD::SETGE; CC2 = ISD::SETO; Opc = ISD::AND; break;
1576 case ISD::SETOLT: CC1 = ISD::SETLT; CC2 = ISD::SETO; Opc = ISD::AND; break;
1577 case ISD::SETOLE: CC1 = ISD::SETLE; CC2 = ISD::SETO; Opc = ISD::AND; break;
1578 case ISD::SETONE: CC1 = ISD::SETNE; CC2 = ISD::SETO; Opc = ISD::AND; break;
1579 case ISD::SETUEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETUO; Opc = ISD::OR; break;
1580 case ISD::SETUGT: CC1 = ISD::SETGT; CC2 = ISD::SETUO; Opc = ISD::OR; break;
1581 case ISD::SETUGE: CC1 = ISD::SETGE; CC2 = ISD::SETUO; Opc = ISD::OR; break;
1582 case ISD::SETULT: CC1 = ISD::SETLT; CC2 = ISD::SETUO; Opc = ISD::OR; break;
1583 case ISD::SETULE: CC1 = ISD::SETLE; CC2 = ISD::SETUO; Opc = ISD::OR; break;
1584 case ISD::SETUNE: CC1 = ISD::SETNE; CC2 = ISD::SETUO; Opc = ISD::OR; break;
1641 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr,
1682 if (V.getOpcode() == ISD::UNDEF)
1703 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1725 assert(Node->getOperand(i).getOpcode() == ISD::UNDEF);
1742 if (V.getOpcode() == ISD::UNDEF)
1748 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1751 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1927 OtherOpcode = isDIV ? ISD::SREM : ISD::SDIV;
1929 OtherOpcode = isDIV ? ISD::UREM : ISD::UDIV;
1952 bool isSigned = Opcode == ISD::SDIVREM;
2026 SDValue Lo = DAG.getNode(ISD::ADD, dl,
2036 Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
2059 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2067 Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
2070 Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
2091 SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0,
2093 SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52);
2094 SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84);
2095 SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr);
2096 SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr);
2097 SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt,
2099 return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub);
2108 SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0);
2112 SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst);
2114 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst);
2115 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr);
2117 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or);
2118 SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt);
2125 Op0, DAG.getConstant(0, MVT::i64), ISD::SETLT);
2126 return DAG.getNode(ISD::SELECT, dl, MVT::f32, SignBitTest, Slow, Fast);
2131 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2133 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And,
2135 SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2138 And2, DAG.getConstant(UINT64_C(0), MVT::i64), ISD::SETNE);
2139 SDValue Sel = DAG.getNode(ISD::SELECT, dl, MVT::i64, Ne, Or, Op0);
2142 ISD::SETUGE);
2143 SDValue Sel2 = DAG.getNode(ISD::SELECT, dl, MVT::i64, Ge, Sel, Op0);
2146 SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2,
2148 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh);
2149 SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc);
2152 SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt);
2153 SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2);
2154 SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo);
2155 SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2);
2156 return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd,
2160 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2164 ISD::SETLT);
2166 SDValue CstOffset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(),
2186 CPIdx = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), CPIdx, CstOffset);
2194 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT,
2203 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2226 if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
2227 OpToUse = ISD::SINT_TO_FP;
2233 if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
2234 OpToUse = ISD::UINT_TO_FP;
2244 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2267 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
2268 OpToUse = ISD::FP_TO_SINT;
2272 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
2273 OpToUse = ISD::FP_TO_UINT;
2286 return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2298 Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT));
2299 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT));
2300 return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
2302 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT));
2303 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT));
2304 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT));
2305 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT));
2306 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(0xFF0000, VT));
2307 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, VT));
2308 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2309 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2310 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2312 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, SHVT));
2313 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, SHVT));
2314 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT));
2315 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT));
2316 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT));
2317 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT));
2318 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, SHVT));
2319 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, SHVT));
2320 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, DAG.getConstant(255ULL<<48, VT));
2321 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, DAG.getConstant(255ULL<<40, VT));
2322 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, DAG.getConstant(255ULL<<32, VT));
2323 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, DAG.getConstant(255ULL<<24, VT));
2324 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(255ULL<<16, VT));
2325 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(255ULL<<8 , VT));
2326 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
2327 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
2328 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2329 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2330 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
2331 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2332 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
2354 case ISD::CTPOP: {
2371 Op = DAG.getNode(ISD::SUB, dl, VT, Op,
2372 DAG.getNode(ISD::AND, dl, VT,
2373 DAG.getNode(ISD::SRL, dl, VT, Op,
2377 Op = DAG.getNode(ISD::ADD, dl, VT,
2378 DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
2379 DAG.getNode(ISD::AND, dl, VT,
2380 DAG.getNode(ISD::SRL, dl, VT, Op,
2384 Op = DAG.getNode(ISD::AND, dl, VT,
2385 DAG.getNode(ISD::ADD, dl, VT, Op,
2386 DAG.getNode(ISD::SRL, dl, VT, Op,
2390 Op = DAG.getNode(ISD::SRL, dl, VT,
2391 DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
2396 case ISD::CTLZ_ZERO_UNDEF:
2398 return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op);
2399 case ISD::CTLZ: {
2414 Op = DAG.getNode(ISD::OR, dl, VT, Op,
2415 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3));
2418 return DAG.getNode(ISD::CTPOP, dl, VT, Op);
2420 case ISD::CTTZ_ZERO_UNDEF:
2422 return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op);
2423 case ISD::CTTZ: {
2429 SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT,
2431 DAG.getNode(ISD::SUB, dl, VT, Op,
2433 // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
2434 if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) &&
2435 TLI.isOperationLegalOrCustom(ISD::CTLZ, VT))
2436 return DAG.getNode(ISD::SUB, dl, VT,
2438 DAG.getNode(ISD::CTLZ, dl, VT, Tmp3));
2439 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3);
2452 case ISD::ATOMIC_SWAP:
2461 case ISD::ATOMIC_CMP_SWAP:
2470 case ISD::ATOMIC_LOAD_ADD:
2479 case ISD::ATOMIC_LOAD_SUB:
2488 case ISD::ATOMIC_LOAD_AND:
2497 case ISD::ATOMIC_LOAD_OR:
2506 case ISD::ATOMIC_LOAD_XOR:
2515 case ISD::ATOMIC_LOAD_NAND:
2534 case ISD::CTPOP:
2535 case ISD::CTLZ:
2536 case ISD::CTLZ_ZERO_UNDEF:
2537 case ISD::CTTZ:
2538 case ISD::CTTZ_ZERO_UNDEF:
2542 case ISD::BSWAP:
2545 case ISD::FRAMEADDR:
2546 case ISD::RETURNADDR:
2547 case ISD::FRAME_TO_ARGS_OFFSET:
2550 case ISD::FLT_ROUNDS_:
2553 case ISD::EH_RETURN:
2554 case ISD::EH_LABEL:
2555 case ISD::PREFETCH:
2556 case ISD::VAEND:
2557 case ISD::EH_SJLJ_LONGJMP:
2562 case ISD::EH_SJLJ_SETJMP:
2568 case ISD::ATOMIC_FENCE:
2569 case ISD::MEMBARRIER: {
2584 case ISD::ATOMIC_LOAD: {
2587 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl,
2598 case ISD::ATOMIC_STORE: {
2600 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2613 case ISD::ATOMIC_SWAP:
2614 case ISD::ATOMIC_LOAD_ADD:
2615 case ISD::ATOMIC_LOAD_SUB:
2616 case ISD::ATOMIC_LOAD_AND:
2617 case ISD::ATOMIC_LOAD_OR:
2618 case ISD::ATOMIC_LOAD_XOR:
2619 case ISD::ATOMIC_LOAD_NAND:
2620 case ISD::ATOMIC_LOAD_MIN:
2621 case ISD::ATOMIC_LOAD_MAX:
2622 case ISD::ATOMIC_LOAD_UMIN:
2623 case ISD::ATOMIC_LOAD_UMAX:
2624 case ISD::ATOMIC_CMP_SWAP: {
2630 case ISD::DYNAMIC_STACKALLOC:
2633 case ISD::MERGE_VALUES:
2637 case ISD::UNDEF: {
2647 case ISD::TRAP: {
2660 case ISD::FP_ROUND:
2661 case ISD::BITCAST:
2666 case ISD::FP_EXTEND:
2672 case ISD::SIGN_EXTEND_INREG: {
2683 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
2685 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
2689 case ISD::FP_ROUND_INREG: {
2702 case ISD::SINT_TO_FP:
2703 case ISD::UINT_TO_FP:
2704 Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
2708 case ISD::FP_TO_UINT: {
2718 Tmp1, ISD::SETLT);
2719 True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0));
2720 False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT,
2721 DAG.getNode(ISD::FSUB, dl, VT,
2723 False = DAG.getNode(ISD::XOR, dl, NVT, False,
2725 Tmp1 = DAG.getNode(ISD::SELECT, dl, NVT, Tmp2, True, False);
2729 case ISD::VAARG: {
2744 VAList = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), VAList,
2748 VAList = DAG.getNode(ISD::AND, dl, TLI.getPointerTy(), VAList,
2754 Tmp3 = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), VAList,
2767 case ISD::VACOPY: {
2780 case ISD::EXTRACT_VECTOR_ELT:
2783 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
2789 case ISD::EXTRACT_SUBVECTOR:
2792 case ISD::INSERT_SUBVECTOR:
2795 case ISD::CONCAT_VECTORS: {
2799 case ISD::SCALAR_TO_VECTOR:
2802 case ISD::INSERT_VECTOR_ELT:
2807 case ISD::VECTOR_SHUFFLE: {
2833 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
2834 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
2866 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
2870 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
2875 Tmp1 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], Ops.size());
2877 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
2881 case ISD::EXTRACT_ELEMENT: {
2885 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
2888 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
2891 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
2897 case ISD::STACKSAVE:
2909 case ISD::STACKRESTORE:
2919 case ISD::FCOPYSIGN:
2922 case ISD::FNEG:
2925 Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
2929 case ISD::FABS: {
2935 Tmp1, Tmp2, ISD::SETUGT);
2936 Tmp3 = DAG.getNode(ISD::FNEG, dl, VT, Tmp1);
2937 Tmp1 = DAG.getNode(ISD::SELECT, dl, VT, Tmp2, Tmp1, Tmp3);
2941 case ISD::FSQRT:
2945 case ISD::FSIN:
2949 case ISD::FCOS:
2953 case ISD::FLOG:
2957 case ISD::FLOG2:
2961 case ISD::FLOG10:
2965 case ISD::FEXP:
2969 case ISD::FEXP2:
2973 case ISD::FTRUNC:
2977 case ISD::FFLOOR:
2981 case ISD::FCEIL:
2985 case ISD::FRINT:
2989 case ISD::FNEARBYINT:
2995 case ISD::FPOWI:
2999 case ISD::FPOW:
3003 case ISD::FDIV:
3007 case ISD::FREM:
3011 case ISD::FMA:
3015 case ISD::FP16_TO_FP32:
3018 case ISD::FP32_TO_FP16:
3021 case ISD::ConstantFP: {
3029 case ISD::EHSELECTION: {
3037 case ISD::EXCEPTIONADDR: {
3045 case ISD::FSUB: {
3047 assert(TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3048 TLI.isOperationLegalOrCustom(ISD::FNEG, VT) &&
3050 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3051 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1);
3055 case ISD::SUB: {
3057 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3058 ISD::XOR, VT) &&
3060 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3062 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp2, DAG.getConstant(1, VT));
3063 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3066 case ISD::UREM:
3067 case ISD::SREM: {
3070 bool isSigned = Node->getOpcode() == ISD::SREM;
3071 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
3072 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3082 Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
3083 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
3097 case ISD::UDIV:
3098 case ISD::SDIV: {
3099 bool isSigned = Node->getOpcode() == ISD::SDIV;
3100 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3121 case ISD::MULHU:
3122 case ISD::MULHS: {
3123 unsigned ExpandOpcode = Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI :
3124 ISD::SMUL_LOHI;
3134 case ISD::SDIVREM:
3135 case ISD::UDIVREM:
3139 case ISD::MUL: {
3147 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3148 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3149 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3150 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3153 OpToUse = ISD::SMUL_LOHI;
3155 OpToUse = ISD::UMUL_LOHI;
3157 OpToUse = ISD::SMUL_LOHI;
3159 OpToUse = ISD::UMUL_LOHI;
3173 case ISD::SADDO:
3174 case ISD::SSUBO: {
3177 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3178 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3194 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3195 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3197 Node->getOpcode() == ISD::SADDO ?
3198 ISD::SETEQ : ISD::SETNE);
3200 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3201 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3203 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3207 case ISD::UADDO:
3208 case ISD::USUBO: {
3211 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ?
3212 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3216 Node->getOpcode () == ISD::UADDO ?
3217 ISD::SETULT : ISD::SETUGT));
3220 case ISD::UMULO:
3221 case ISD::SMULO: {
3229 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
3230 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
3231 bool isSigned = Node->getOpcode() == ISD::SMULO;
3233 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3243 Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
3244 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3246 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3267 SDValue HiLHS = DAG.getNode(ISD::SRA, dl, VT, RHS,
3269 SDValue HiRHS = DAG.getNode(ISD::SRA, dl, VT, LHS,
3278 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret,
3280 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret,
3291 Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
3293 ISD::SETNE);
3296 DAG.getConstant(0, VT), ISD::SETNE);
3302 case ISD::BUILD_PAIR: {
3304 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3305 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3306 Tmp2 = DAG.getNode(ISD::SHL, dl, PairTy, Tmp2,
3309 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3312 case ISD::SELECT:
3316 if (Tmp1.getOpcode() == ISD::SETCC) {
3323 Tmp2, Tmp3, ISD::SETNE);
3327 case ISD::BR_JT: {
3338 Index = DAG.getNode(ISD::MUL, dl, PTy,
3340 SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Index, Table);
3343 SDValue LD = DAG.getExtLoad(ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3351 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3354 Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr);
3358 case ISD::BRCOND:
3363 if (Tmp2.getOpcode() == ISD::SETCC) {
3364 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3370 Tmp3 = (Tmp2.getOpcode() == ISD::UNDEF) ? Tmp2 :
3371 DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3373 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3374 DAG.getCondCode(ISD::SETNE), Tmp3,
3380 case ISD::SETCC: {
3395 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3400 case ISD::SELECT_CC: {
3412 CC = DAG.getCondCode(ISD::SETNE);
3413 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, Tmp2,
3418 case ISD::BR_CC: {
3429 Tmp4 = DAG.getCondCode(ISD::SETNE);
3430 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, Tmp2,
3435 case ISD::BUILD_VECTOR:
3438 case ISD::SRA:
3439 case ISD::SRL:
3440 case ISD::SHL: {
3449 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
3452 SDValue Sh = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
3459 DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0),
3464 case ISD::GLOBAL_OFFSET_TABLE:
3465 case ISD::GlobalAddress:
3466 case ISD::GlobalTLSAddress:
3467 case ISD::ExternalSymbol:
3468 case ISD::ConstantPool:
3469 case ISD::JumpTable:
3470 case ISD::INTRINSIC_W_CHAIN:
3471 case ISD::INTRINSIC_WO_CHAIN:
3472 case ISD::INTRINSIC_VOID:
3485 if (Node->getOpcode() == ISD::UINT_TO_FP ||
3486 Node->getOpcode() == ISD::SINT_TO_FP ||
3487 Node->getOpcode() == ISD::SETCC) {
3494 case ISD::CTTZ:
3495 case ISD::CTTZ_ZERO_UNDEF:
3496 case ISD::CTLZ:
3497 case ISD::CTLZ_ZERO_UNDEF:
3498 case ISD::CTPOP:
3500 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
3504 if (Node->getOpcode() == ISD::CTTZ) {
3508 ISD::SETEQ);
3509 Tmp1 = DAG.getNode(ISD::SELECT, dl, NVT, Tmp2,
3511 } else if (Node->getOpcode() == ISD::CTLZ ||
3512 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
3514 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
3518 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
3520 case ISD::BSWAP: {
3522 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
3523 Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1);
3524 Tmp1 = DAG.getNode(ISD::SRL, dl, NVT, Tmp1,
3529 case ISD::FP_TO_UINT:
3530 case ISD::FP_TO_SINT:
3532 Node->getOpcode() == ISD::FP_TO_SINT, dl);
3535 case ISD::UINT_TO_FP:
3536 case ISD::SINT_TO_FP:
3538 Node->getOpcode() == ISD::SINT_TO_FP, dl);
3541 case ISD::VAARG: {
3547 TruncOp = ISD::BITCAST;
3551 TruncOp = ISD::TRUNCATE;
3568 case ISD::AND:
3569 case ISD::OR:
3570 case ISD::XOR: {
3573 ExtOp = ISD::BITCAST;
3574 TruncOp = ISD::BITCAST;
3577 ExtOp = ISD::ANY_EXTEND;
3578 TruncOp = ISD::TRUNCATE;
3588 case ISD::SELECT: {
3591 ExtOp = ISD::BITCAST;
3592 TruncOp = ISD::BITCAST;
3594 ExtOp = ISD::ANY_EXTEND;
3595 TruncOp = ISD::TRUNCATE;
3597 ExtOp = ISD::FP_EXTEND;
3598 TruncOp = ISD::FP_ROUND;
3605 Tmp1 = DAG.getNode(ISD::SELECT, dl, NVT, Tmp1, Tmp2, Tmp3);
3606 if (TruncOp != ISD::FP_ROUND)
3614 case ISD::VECTOR_SHUFFLE: {
3618 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
3619 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
3623 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
3627 case ISD::SETCC: {
3628 unsigned ExtOp = ISD::FP_EXTEND;
3630 ISD::CondCode CCCode =
3632 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3636 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3640 case ISD::FDIV:
3641 case ISD::FREM:
3642 case ISD::FPOW: {
3643 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
3644 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
3646 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
3650 case ISD::FLOG2:
3651 case ISD::FEXP2:
3652 case ISD::FLOG:
3653 case ISD::FEXP: {
3654 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
3656 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
