Lines Matching refs:ISD
273 TLI.isLoadExtLegal(ISD::EXTLOAD, SVT) &&
286 DAG.getExtLoad(ISD::EXTLOAD, dl, OrigVT,
303 assert(ST->getAddressingMode() == ISD::UNINDEXED &&
318 SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val);
359 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
361 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
371 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Store, StackPtr,
383 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Stores[0],
400 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);
407 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
416 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
425 assert(LD->getAddressingMode() == ISD::UNINDEXED &&
441 SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad);
443 Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND :
444 ISD::ANY_EXTEND, dl, VT, Result);
479 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
480 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
487 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr,
500 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Stores[0],
524 ISD::LoadExtType HiExtType = LD->getExtensionType();
527 if (HiExtType == ISD::NON_EXTLOAD)
528 HiExtType = ISD::ZEXTLOAD;
533 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(),
536 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
546 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
548 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
557 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);
558 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo);
560 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
598 unsigned CastOpc = IdxVT.bitsGT(PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND;
602 Tmp3 = DAG.getNode(ISD::MUL, dl, IdxVT, Tmp3,DAG.getConstant(EltSize, IdxVT));
603 SDValue StackPtr2 = DAG.getNode(ISD::ADD, dl, IdxVT, Tmp3, StackPtr);
623 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
685 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
691 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
717 switch (TLI.getOperationAction(ISD::STORE, VT)) {
737 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
740 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
791 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
793 Hi = DAG.getNode(ISD::SRL, dl, Value.getValueType(), Value,
804 Hi = DAG.getNode(ISD::SRL, dl, Value.getValueType(), Value,
812 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
821 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
850 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
868 ISD::LoadExtType ExtType = LD->getExtensionType();
869 if (ExtType == ISD::NON_EXTLOAD) {
904 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
943 ISD::LoadExtType NewExtType =
944 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
953 if (ExtType == ISD::SEXTLOAD)
955 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
958 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
960 Result = DAG.getNode(ISD::AssertZext, dl,
983 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0),
990 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
999 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1003 Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi,
1008 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
1019 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
1021 Lo = DAG.getExtLoad(ISD::ZEXTLOAD,
1029 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1033 Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi,
1038 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
1076 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, SrcVT) && TLI.isTypeLegal(SrcVT)) {
1083 case ISD::EXTLOAD:
1085 ISD::FP_EXTEND : ISD::ANY_EXTEND);
1087 case ISD::SEXTLOAD: ExtendOp = ISD::SIGN_EXTEND; break;
1088 case ISD::ZEXTLOAD: ExtendOp = ISD::ZERO_EXTEND; break;
1103 assert(ExtType != ISD::EXTLOAD &&
1107 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, Node->getValueType(0),
1112 if (ExtType == ISD::SEXTLOAD)
1113 ValRes = DAG.getNode(ISD
1137 if (Node->getOpcode() == ISD::TargetConstant) // Allow illegal target nodes.
1149 Node->getOperand(i).getOpcode() == ISD::TargetConstant) &&
1156 case ISD::INTRINSIC_W_CHAIN:
1157 case ISD::INTRINSIC_WO_CHAIN:
1158 case ISD::INTRINSIC_VOID:
1159 case ISD::STACKSAVE:
1162 case ISD::VAARG:
1168 case ISD::SINT_TO_FP:
1169 case ISD::UINT_TO_FP:
1170 case ISD::EXTRACT_VECTOR_ELT:
1174 case ISD::FP_ROUND_INREG:
1175 case ISD::SIGN_EXTEND_INREG: {
1180 case ISD::ATOMIC_STORE: {
1185 case ISD::SELECT_CC:
1186 case ISD::SETCC:
1187 case ISD::BR_CC: {
1188 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
1189 Node->getOpcode() == ISD::SETCC ? 2 : 1;
1190 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
1192 ISD::CondCode CCCode =
1196 if (Node->getOpcode() == ISD::SELECT_CC)
1204 case ISD::LOAD:
1205 case ISD::STORE:
1210 case ISD::CALLSEQ_START:
1211 case ISD::CALLSEQ_END:
1217 case ISD::EXTRACT_ELEMENT:
1218 case ISD::FLT_ROUNDS_:
1219 case ISD::SADDO:
1220 case ISD::SSUBO:
1221 case ISD::UADDO:
1222 case ISD::USUBO:
1223 case ISD::SMULO:
1224 case ISD::UMULO:
1225 case ISD::FPOWI:
1226 case ISD::MERGE_VALUES:
1227 case ISD::EH_RETURN:
1228 case ISD::FRAME_TO_ARGS_OFFSET:
1229 case ISD::EH_SJLJ_SETJMP:
1230 case ISD::EH_SJLJ_LONGJMP:
1237 case ISD::INIT_TRAMPOLINE:
1238 case ISD::ADJUST_TRAMPOLINE:
1239 case ISD::FRAMEADDR:
1240 case ISD::RETURNADDR:
1247 case ISD::DEBUGTRAP:
1250 // replace ISD::DEBUGTRAP with ISD::TRAP
1252 NewVal = DAG.getNode(ISD::TRAP, Node->getDebugLoc(), Node->getVTList(),
1261 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1273 case ISD::SHL:
1274 case ISD::SRL:
1275 case ISD::SRA:
1276 case ISD::ROTL:
1277 case ISD::ROTR:
1290 case ISD::SRL_PARTS:
1291 case ISD::SRA_PARTS:
1292 case ISD::SHL_PARTS:
1358 case ISD::CALLSEQ_START:
1359 case ISD::CALLSEQ_END:
1361 case ISD::LOAD: {
1364 case ISD::STORE: {
1382 Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx,
1386 Idx = DAG.getNode(ISD::TRUNCATE, dl, TLI.getPointerTy(), Idx);
1388 Idx = DAG.getNode(ISD::ZERO_EXTEND, dl, TLI.getPointerTy(), Idx);
1390 StackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, StackPtr);
1395 return DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1425 Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx,
1429 Idx = DAG.getNode(ISD::TRUNCATE, dl, TLI.getPointerTy(), Idx);
1431 Idx = DAG.getNode(ISD::ZERO_EXTEND, dl, TLI.getPointerTy(), Idx);
1433 SDValue SubStackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx,
1463 if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue;
1468 Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
1486 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1508 SignBit = DAG.getNode(ISD::BITCAST, dl, IVT, Tmp2);
1529 LoadPtr = DAG.getNode(ISD::ADD, dl, LoadPtr.getValueType(),
1539 SignBit = DAG.getNode(ISD::SHL, dl, LoadTy, SignBit,
1547 ISD::SETLT);
1549 SDValue AbsVal = DAG.getNode(ISD::FABS, dl, Tmp1.getValueType(), Tmp1);
1552 return DAG.getNode(ISD::SELECT, dl, AbsVal.getValueType(), SignBit,
1553 DAG.getNode(ISD::FNEG, dl, AbsVal.getValueType(), AbsVal),
1579 SP = DAG.getNode(ISD::AND, dl, VT, SP,
1581 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
1599 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1606 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
1607 ISD::CondCode InvCC = ISD::SETCC_INVALID;
1611 case ISD::SETO:
1612 assert(TLI.getCondCodeAction(ISD::SETOEQ, OpVT)
1615 CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
1616 case ISD::SETUO:
1617 assert(TLI.getCondCodeAction(ISD::SETUNE, OpVT)
1620 CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break;
1621 case ISD::SETOEQ:
1622 case ISD::SETOGT:
1623 case ISD::SETOGE:
1624 case ISD::SETOLT:
1625 case ISD::SETOLE:
1626 case ISD::SETONE:
1627 case ISD::SETUEQ:
1628 case ISD::SETUNE:
1629 case ISD::SETUGT:
1630 case ISD::SETUGE:
1631 case ISD::SETULT:
1632 case ISD::SETULE:
1637 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
1638 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
1639 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
1643 case ISD::SETLE:
1644 case ISD::SETGT:
1645 case ISD::SETGE:
1646 case ISD::SETLT:
1647 case ISD::SETNE:
1648 case ISD::SETEQ:
1649 InvCC = ISD::getSetCCSwappedOperands(CCCode);
1662 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
1723 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr,
1764 if (V.getOpcode() == ISD::UNDEF)
1785 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1807 assert(Node->getOperand(i).getOpcode() == ISD::UNDEF);
1824 if (V.getOpcode() == ISD::UNDEF)
1830 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1833 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
2015 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
2018 OtherOpcode = isDIV ? ISD::SREM : ISD::SDIV;
2020 OtherOpcode = isDIV ? ISD::UREM : ISD::UDIV;
2043 bool isSigned = Opcode == ISD::SDIVREM;
2131 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2132 ? ISD::FCOS : ISD::FSIN;
2141 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2232 SDValue Lo = DAG.getNode(ISD::ADD, dl,
2242 Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
2265 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2273 Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
2276 Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
2297 SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0,
2299 SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52);
2300 SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84);
2301 SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr);
2302 SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr);
2303 SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt,
2305 return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub);
2314 SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0);
2318 SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst);
2320 ISD::AND, dl, MVT::i64, Op0, AndConst);
2321 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr);
2323 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or);
2324 SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt);
2331 Op0, DAG.getConstant(0, MVT::i64), ISD::SETLT);
2332 return DAG.getNode(ISD::SELECT, dl, MVT::f32, SignBitTest, Slow, Fast);
2337 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2339 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And,
2341 SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2344 And2, DAG.getConstant(UINT64_C(0), MVT::i64), ISD::SETNE);
2345 SDValue Sel = DAG.getNode(ISD::SELECT, dl, MVT::i64, Ne, Or, Op0);
2348 ISD::SETUGE);
2349 SDValue Sel2 = DAG.getNode(ISD::SELECT, dl, MVT::i64, Ge, Sel, Op0);
2352 SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2,
2354 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh);
2355 SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc);
2358 SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt);
2359 SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2);
2360 SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo);
2361 SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2);
2362 return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd,
2366 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2370 ISD::SETLT);
2372 SDValue CstOffset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(),
2392 CPIdx = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), CPIdx, CstOffset);
2400 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT,
2409 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2432 if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
2433 OpToUse = ISD::SINT_TO_FP;
2439 if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
2440 OpToUse = ISD::UINT_TO_FP;
2450 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2473 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
2474 OpToUse = ISD::FP_TO_SINT;
2478 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
2479 OpToUse = ISD::FP_TO_UINT;
2492 return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2504 Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT));
2505 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT));
2506 return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
2508 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT));
2509 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT));
2510 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT));
2511 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT));
2512 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(0xFF0000, VT));
2513 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, VT));
2514 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2515 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2516 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2518 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, SHVT));
2519 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, SHVT));
2520 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT));
2521 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT));
2522 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT));
2523 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT));
2524 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, SHVT));
2525 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, SHVT));
2526 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, DAG.getConstant(255ULL<<48, VT));
2527 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, DAG.getConstant(255ULL<<40, VT));
2528 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, DAG.getConstant(255ULL<<32, VT));
2529 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, DAG.getConstant(255ULL<<24, VT));
2530 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(255ULL<<16, VT));
2531 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(255ULL<<8 , VT));
2532 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
2533 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
2534 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2535 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2536 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
2537 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2538 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
2548 case ISD::CTPOP: {
2565 Op = DAG.getNode(ISD::SUB, dl, VT, Op,
2566 DAG.getNode(ISD::AND, dl, VT,
2567 DAG.getNode(ISD::SRL, dl, VT, Op,
2571 Op = DAG.getNode(ISD::ADD, dl, VT,
2572 DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
2573 DAG.getNode(ISD::AND, dl, VT,
2574 DAG.getNode(ISD::SRL, dl, VT, Op,
2578 Op = DAG.getNode(ISD::AND, dl, VT,
2579 DAG.getNode(ISD::ADD, dl, VT, Op,
2580 DAG.getNode(ISD::SRL, dl, VT, Op,
2584 Op = DAG.getNode(ISD::SRL, dl, VT,
2585 DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
2590 case ISD::CTLZ_ZERO_UNDEF:
2592 return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op);
2593 case ISD::CTLZ: {
2608 Op = DAG.getNode(ISD::OR, dl, VT, Op,
2609 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3));
2612 return DAG.getNode(ISD::CTPOP, dl, VT, Op);
2614 case ISD::CTTZ_ZERO_UNDEF:
2616 return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op);
2617 case ISD::CTTZ: {
2623 SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT,
2625 DAG.getNode(ISD::SUB, dl, VT, Op,
2627 // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
2628 if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) &&
2629 TLI.isOperationLegalOrCustom(ISD::CTLZ, VT))
2630 return DAG.getNode(ISD::SUB, dl, VT,
2632 DAG.getNode(ISD::CTLZ, dl, VT, Tmp3));
2633 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3);
2646 case ISD::ATOMIC_SWAP:
2655 case ISD::ATOMIC_CMP_SWAP:
2664 case ISD::ATOMIC_LOAD_ADD:
2673 case ISD::ATOMIC_LOAD_SUB:
2682 case ISD::ATOMIC_LOAD_AND:
2691 case ISD::ATOMIC_LOAD_OR:
2700 case ISD::ATOMIC_LOAD_XOR:
2709 case ISD::ATOMIC_LOAD_NAND:
2728 case ISD::CTPOP:
2729 case ISD::CTLZ:
2730 case ISD::CTLZ_ZERO_UNDEF:
2731 case ISD::CTTZ:
2732 case ISD::CTTZ_ZERO_UNDEF:
2736 case ISD::BSWAP:
2739 case ISD::FRAMEADDR:
2740 case ISD::RETURNADDR:
2741 case ISD::FRAME_TO_ARGS_OFFSET:
2744 case ISD::FLT_ROUNDS_:
2747 case ISD::EH_RETURN:
2748 case ISD::EH_LABEL:
2749 case ISD::PREFETCH:
2750 case ISD::VAEND:
2751 case ISD::EH_SJLJ_LONGJMP:
2756 case ISD::EH_SJLJ_SETJMP:
2762 case ISD::ATOMIC_FENCE:
2763 case ISD::MEMBARRIER: {
2781 case ISD::ATOMIC_LOAD: {
2784 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl,
2795 case ISD::ATOMIC_STORE: {
2797 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2810 case ISD::ATOMIC_SWAP:
2811 case ISD::ATOMIC_LOAD_ADD:
2812 case ISD::ATOMIC_LOAD_SUB:
2813 case ISD::ATOMIC_LOAD_AND:
2814 case ISD::ATOMIC_LOAD_OR:
2815 case ISD::ATOMIC_LOAD_XOR:
2816 case ISD::ATOMIC_LOAD_NAND:
2817 case ISD::ATOMIC_LOAD_MIN:
2818 case ISD::ATOMIC_LOAD_MAX:
2819 case ISD::ATOMIC_LOAD_UMIN:
2820 case ISD::ATOMIC_LOAD_UMAX:
2821 case ISD::ATOMIC_CMP_SWAP: {
2827 case ISD::DYNAMIC_STACKALLOC:
2830 case ISD::MERGE_VALUES:
2834 case ISD::UNDEF: {
2844 case ISD::TRAP: {
2860 case ISD::FP_ROUND:
2861 case ISD::BITCAST:
2866 case ISD::FP_EXTEND:
2872 case ISD::SIGN_EXTEND_INREG: {
2883 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
2885 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
2889 case ISD::FP_ROUND_INREG: {
2902 case ISD::SINT_TO_FP:
2903 case ISD::UINT_TO_FP:
2904 Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
2908 case ISD::FP_TO_UINT: {
2919 Tmp1, ISD::SETLT);
2920 True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0));
2921 False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT,
2922 DAG.getNode(ISD::FSUB, dl, VT,
2924 False = DAG.getNode(ISD::XOR, dl, NVT, False,
2926 Tmp1 = DAG.getNode(ISD::SELECT, dl, NVT, Tmp2, True, False);
2930 case ISD::VAARG: {
2945 VAList = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), VAList,
2949 VAList = DAG.getNode(ISD::AND, dl, TLI.getPointerTy(), VAList,
2955 Tmp3 = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), VAList,
2968 case ISD::VACOPY: {
2981 case ISD::EXTRACT_VECTOR_ELT:
2984 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
2990 case ISD::EXTRACT_SUBVECTOR:
2993 case ISD::INSERT_SUBVECTOR:
2996 case ISD::CONCAT_VECTORS: {
3000 case ISD::SCALAR_TO_VECTOR:
3003 case ISD::INSERT_VECTOR_ELT:
3008 case ISD::VECTOR_SHUFFLE: {
3034 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3035 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3067 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
3071 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
3076 Tmp1 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], Ops.size());
3078 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3082 case ISD::EXTRACT_ELEMENT: {
3086 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3089 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3092 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3098 case ISD::STACKSAVE:
3110 case ISD::STACKRESTORE:
3120 case ISD::FCOPYSIGN:
3123 case ISD::FNEG:
3126 Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
3130 case ISD::FABS: {
3136 Tmp1, Tmp2, ISD::SETUGT);
3137 Tmp3 = DAG.getNode(ISD::FNEG, dl, VT, Tmp1);
3138 Tmp1 = DAG.getNode(ISD::SELECT, dl, VT, Tmp2, Tmp1, Tmp3);
3142 case ISD::FSQRT:
3147 case ISD::FSIN:
3148 case ISD::FCOS: {
3150 bool isSIN = Node->getOpcode() == ISD::FSIN;
3151 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3153 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3157 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3172 case ISD::FSINCOS:
3176 case ISD::FLOG:
3181 case ISD::FLOG2:
3186 case ISD::FLOG10:
3191 case ISD::FEXP:
3196 case ISD::FEXP2:
3201 case ISD::FTRUNC:
3206 case ISD::FFLOOR:
3211 case ISD::FCEIL:
3216 case ISD::FRINT:
3221 case ISD::FNEARBYINT:
3228 case ISD::FPOWI:
3233 case ISD::FPOW:
3238 case ISD::FDIV:
3243 case ISD::FREM:
3248 case ISD::FMA:
3253 case ISD::FP16_TO_FP32:
3256 case ISD::FP32_TO_FP16:
3259 case ISD::ConstantFP: {
3267 case ISD::EHSELECTION: {
3275 case ISD::EXCEPTIONADDR: {
3283 case ISD::FSUB: {
3285 assert(TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3286 TLI.isOperationLegalOrCustom(ISD::FNEG, VT) &&
3288 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3289 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1);
3293 case ISD::SUB: {
3295 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3296 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3298 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3300 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, VT));
3301 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3304 case ISD::UREM:
3305 case ISD::SREM: {
3307 bool isSigned = Node->getOpcode() == ISD::SREM;
3308 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
3309 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3322 Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
3323 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
3337 case ISD::UDIV:
3338 case ISD::SDIV: {
3339 bool isSigned = Node->getOpcode() == ISD::SDIV;
3340 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3361 case ISD::MULHU:
3362 case ISD::MULHS: {
3363 unsigned ExpandOpcode = Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI :
3364 ISD::SMUL_LOHI;
3374 case ISD::SDIVREM:
3375 case ISD::UDIVREM:
3379 case ISD::MUL: {
3387 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3388 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3389 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3390 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3393 OpToUse = ISD::SMUL_LOHI;
3395 OpToUse = ISD::UMUL_LOHI;
3397 OpToUse = ISD::SMUL_LOHI;
3399 OpToUse = ISD::UMUL_LOHI;
3413 case ISD::SADDO:
3414 case ISD::SSUBO: {
3417 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3418 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3434 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3435 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3437 Node->getOpcode() == ISD::SADDO ?
3438 ISD::SETEQ : ISD::SETNE);
3440 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3441 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3443 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3447 case ISD::UADDO:
3448 case ISD::USUBO: {
3451 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ?
3452 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3456 Node->getOpcode () == ISD::UADDO ?
3457 ISD::SETULT : ISD::SETUGT));
3460 case ISD::UMULO:
3461 case ISD::SMULO: {
3469 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
3470 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
3471 bool isSigned = Node->getOpcode() == ISD::SMULO;
3473 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3483 Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
3484 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3486 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3507 SDValue HiLHS = DAG.getNode(ISD::SRA, dl, VT, RHS,
3509 SDValue HiRHS = DAG.getNode(ISD::SRA, dl, VT, LHS,
3518 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret,
3520 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret,
3531 Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
3533 ISD::SETNE);
3536 DAG.getConstant(0, VT), ISD::SETNE);
3542 case ISD::BUILD_PAIR: {
3544 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3545 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3546 Tmp2 = DAG.getNode(ISD::SHL, dl, PairTy, Tmp2,
3549 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3552 case ISD::SELECT:
3556 if (Tmp1.getOpcode() == ISD::SETCC) {
3563 Tmp2, Tmp3, ISD::SETNE);
3567 case ISD::BR_JT: {
3578 Index = DAG.getNode(ISD::MUL, dl, PTy,
3580 SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Index, Table);
3583 SDValue LD = DAG.getExtLoad(ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3591 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3594 Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr);
3598 case ISD::BRCOND:
3603 if (Tmp2.getOpcode() == ISD::SETCC) {
3604 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3610 Tmp3 = (Tmp2.getOpcode() == ISD::UNDEF) ? Tmp2 :
3611 DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3613 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3614 DAG.getCondCode(ISD::SETNE), Tmp3,
3620 case ISD::SETCC: {
3645 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3651 case ISD::SELECT_CC: {
3663 CC = DAG.getCondCode(ISD::SETNE);
3664 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, Tmp2,
3669 case ISD::BR_CC: {
3680 Tmp4 = DAG.getCondCode(ISD::SETNE);
3681 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, Tmp2,
3686 case ISD::BUILD_VECTOR:
3689 case ISD::SRA:
3690 case ISD::SRL:
3691 case ISD::SHL: {
3700 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
3703 SDValue Sh = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
3710 DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0),
3715 case ISD::GLOBAL_OFFSET_TABLE:
3716 case ISD::GlobalAddress:
3717 case ISD::GlobalTLSAddress:
3718 case ISD::ExternalSymbol:
3719 case ISD::ConstantPool:
3720 case ISD::JumpTable:
3721 case ISD::INTRINSIC_W_CHAIN:
3722 case ISD::INTRINSIC_WO_CHAIN:
3723 case ISD::INTRINSIC_VOID:
3736 if (Node->getOpcode() == ISD::UINT_TO_FP ||
3737 Node->getOpcode() == ISD::SINT_TO_FP ||
3738 Node->getOpcode() == ISD::SETCC) {
3745 case ISD::CTTZ:
3746 case ISD::CTTZ_ZERO_UNDEF:
3747 case ISD::CTLZ:
3748 case ISD::CTLZ_ZERO_UNDEF:
3749 case ISD::CTPOP:
3751 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
3755 if (Node->getOpcode() == ISD::CTTZ) {
3759 ISD::SETEQ);
3760 Tmp1 = DAG.getNode(ISD::SELECT, dl, NVT, Tmp2,
3762 } else if (Node->getOpcode() == ISD::CTLZ ||
3763 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
3765 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
3769 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
3771 case ISD::BSWAP: {
3773 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
3774 Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1);
3775 Tmp1 = DAG.getNode(ISD::SRL, dl, NVT, Tmp1,
3780 case ISD::FP_TO_UINT:
3781 case ISD::FP_TO_SINT:
3783 Node->getOpcode() == ISD::FP_TO_SINT, dl);
3786 case ISD::UINT_TO_FP:
3787 case ISD::SINT_TO_FP:
3789 Node->getOpcode() == ISD::SINT_TO_FP, dl);
3792 case ISD::VAARG: {
3798 TruncOp = ISD::BITCAST;
3802 TruncOp = ISD::TRUNCATE;
3819 case ISD::AND:
3820 case ISD::OR:
3821 case ISD::XOR: {
3824 ExtOp = ISD::BITCAST;
3825 TruncOp = ISD::BITCAST;
3828 ExtOp = ISD::ANY_EXTEND;
3829 TruncOp = ISD::TRUNCATE;
3839 case ISD::SELECT: {
3842 ExtOp = ISD::BITCAST;
3843 TruncOp = ISD::BITCAST;
3845 ExtOp = ISD::ANY_EXTEND;
3846 TruncOp = ISD::TRUNCATE;
3848 ExtOp = ISD::FP_EXTEND;
3849 TruncOp = ISD::FP_ROUND;
3856 Tmp1 = DAG.getNode(ISD::SELECT, dl, NVT, Tmp1, Tmp2, Tmp3);
3857 if (TruncOp != ISD::FP_ROUND)
3865 case ISD::VECTOR_SHUFFLE: {
3869 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
3870 Tmp2 = DAG.getNode(ISD
3874 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
3878 case ISD::SETCC: {
3879 unsigned ExtOp = ISD::FP_EXTEND;
3881 ISD::CondCode CCCode =
3883 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3887 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3891 case ISD::FDIV:
3892 case ISD::FREM:
3893 case ISD::FPOW: {
3894 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
3895 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
3897 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
3901 case ISD::FLOG2:
3902 case ISD::FEXP2:
3903 case ISD::FLOG:
3904 case ISD::FEXP: {
3905 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
3907 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
