Lines Matching refs:Node
64 /// reference to it in order to update as necessary on node deletion.
84 void LegalizeOp(SDNode *Node);
89 void LegalizeLoadOps(SDNode *Node);
90 void LegalizeStoreOps(SDNode *Node);
112 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
117 SDNode *Node, bool isSigned);
118 SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
122 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
128 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
129 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
132 SDValue ExpandBUILD_VECTOR(SDNode *Node);
133 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
134 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
139 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
140 SDValue ExpandFABS(SDNode *Node) const;
154 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
160 bool ExpandNode(SDNode *Node);
161 void ConvertNodeToLibcall(SDNode *Node);
162 void PromoteNode(SDNode *Node);
165 // Node replacement helpers
176 "Replacing one node with another that produces a different number "
244 /// Expands the ConstantFP node to an integer constant or
298 /// Expands the Constant node to a load from the constant pool.
528 // Callers expect a MERGE_VALUES node.
728 void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
729 StoreSDNode *ST = cast<StoreSDNode>(Node);
732 SDLoc dl(Node);
758 ExpandUnalignedStore(cast<StoreSDNode>(Node), DAG, TLI, this);
762 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
763 if (Res && Res != SDValue(Node, 0))
764 ReplaceNode(SDValue(Node, 0), Res);
776 ReplaceNode(SDValue(Node, 0), Result);
799 ReplaceNode(SDValue(Node, 0), Result);
860 ReplaceNode(SDValue(Node, 0), Result);
871 ExpandUnalignedStore(cast<StoreSDNode>(Node), DAG, TLI, this);
875 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
876 if (Res && Res != SDValue(Node, 0))
877 ReplaceNode(SDValue(Node, 0), Res);
891 ReplaceNode(SDValue(Node, 0), Result);
898 void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
899 LoadSDNode *LD = cast<LoadSDNode>(Node);
903 SDLoc dl(Node);
907 MVT VT = Node->getSimpleValueType(0);
908 SDValue RVal = SDValue(Node, 0);
909 SDValue RChain = SDValue(Node, 1);
911 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
921 ExpandUnalignedLoad(cast<LoadSDNode>(Node), DAG, TLI, RVal, RChain);
933 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
943 if (RChain.getNode() != Node) {
944 assert(RVal.getNode() != Node && "Load must be completely replaced");
945 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
946 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
951 ReplacedNode(Node);
973 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
988 DAG.getExtLoad(NewExtType, dl, Node->getValueType(0),
1026 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0),
1036 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
1041 // Build a factor node to remember that this load is independent of
1053 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
1058 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
1068 dl, Node->getValueType(0), Chain, Ptr,
1073 // Build a factor node to remember that this load is independent of
1085 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
1091 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
1098 Value = SDValue(Node, 0);
1099 Chain = SDValue(Node, 1);
1102 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
1115 Node), DAG, TLI, Value, Chain);
1120 EVT DestVT = Node->getValueType(0);
1136 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
1171 Node->getValueType(0),
1189 if (Chain.getNode() != Node) {
1190 assert(Value.getNode() != Node && "Load must be completely replaced");
1191 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
1192 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
1197 ReplacedNode(Node);
1202 void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
1203 DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
1205 if (Node->getOpcode() == ISD::TargetConstant) // Allow illegal target nodes.
1209 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
1210 assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
1212 TLI.isTypeLegal(Node->getValueType(i))) &&
1215 for (const SDValue &Op : Node->op_values())
1226 switch (Node->getOpcode()) {
1231 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1234 Action = TLI.getOperationAction(Node->getOpcode(),
1235 Node->getValueType(0));
1238 Action = TLI.getOperationAction(Node->getOpcode(),
1239 Node->getValueType(0));
1241 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1247 Action = TLI.getOperationAction(Node->getOpcode(),
1248 Node->getOperand(0).getValueType());
1252 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1253 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1257 Action = TLI.getOperationAction(Node->getOpcode(),
1258 Node->getOperand(2).getValueType());
1264 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
1265 Node->getOpcode() == ISD::SETCC ? 2 :
1266 Node->getOpcode() == ISD::SETCCE ? 3 : 1;
1267 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
1268 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1270 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1273 if (Node->getOpcode() == ISD::SELECT_CC)
1274 Action = TLI.getOperationAction(Node->getOpcode(),
1275 Node->getValueType(0));
1277 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1305 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1315 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1322 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1332 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1336 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1337 Node->getOperand(0));
1338 ReplaceNode(Node, NewVal.getNode());
1345 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1348 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1354 SDNode *NewNode = Node;
1355 switch (Node->getOpcode()) {
1364 if (!Node->getOperand(1).getValueType().isVector()) {
1366 DAG.getShiftAmountOperand(Node->getOperand(0).getValueType(),
1367 Node->getOperand(1));
1370 NewNode = DAG.UpdateNodeOperands(Node, Node->getOperand(0),
1379 if (!Node->getOperand(2).getValueType().isVector()) {
1381 DAG.getShiftAmountOperand(Node->getOperand(0).getValueType(),
1382 Node->getOperand(2));
1385 NewNode = DAG.UpdateNodeOperands(Node, Node->getOperand(0),
1386 Node->getOperand(1),
1392 if (NewNode != Node) {
1393 ReplaceNode(Node, NewNode);
1394 Node = NewNode;
1402 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
1404 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1407 if (Node->getNumValues() == 1) {
1408 // We can just directly replace this node with the lowered value.
1409 ReplaceNode(SDValue(Node, 0), Res);
1414 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
1416 ReplaceNode(Node, ResultVals.data());
1422 if (ExpandNode(Node))
1426 ConvertNodeToLibcall(Node);
1429 PromoteNode(Node);
1434 switch (Node->getOpcode()) {
1437 dbgs() << "NODE: ";
1438 Node->dump( &DAG);
1447 return LegalizeLoadOps(Node);
1450 return LegalizeStoreOps(Node);
1577 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1582 EVT VT = Node->getValueType(0);
1584 SDLoc dl(Node);
1594 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1596 if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue;
1605 if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
1607 Node->getOperand(i), Idx,
1612 Node->getOperand(i), Idx,
1709 SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1710 SDLoc DL(Node);
1711 SDValue Mag = Node->getOperand(0);
1712 SDValue Sign = Node->getOperand(1);
1746 SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1747 SDLoc DL(Node);
1748 SDValue Value = Node->getOperand(0);
1767 void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1772 SDLoc dl(Node);
1773 EVT VT = Node->getValueType(0);
1774 SDValue Tmp1 = SDValue(Node, 0);
1775 SDValue Tmp2 = SDValue(Node, 1);
1776 SDValue Tmp3 = Node->getOperand(2);
1805 /// If the SETCC has been legalized using AND / OR, then the legalized node
1964 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1965 SDLoc dl(Node);
1968 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1974 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1976 Node->getValueType(0).getVectorElementType(), false, false, 0);
1978 Node->getValueType(0), dl, Ch, StackPtr,
1984 ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1986 unsigned NumElems = Node->getNumOperands();
1987 SDLoc dl(Node);
1988 EVT VT = Node->getValueType(0);
2001 SDValue V = Node->getOperand(i);
2077 /// Expand a BUILD_VECTOR node on targets that don't
2079 SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
2080 unsigned NumElems = Node->getNumOperands();
2082 SDLoc dl(Node);
2083 EVT VT = Node->getValueType(0);
2084 EVT OpVT = Node->getOperand(0).getValueType();
2088 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
2093 SDValue V = Node->getOperand(i);
2115 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
2122 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
2125 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
2137 assert(Node->getOperand(i).getOpcode() == ISD::UNDEF);
2154 if (Node->getOperand(i).getOpcode() == ISD::UNDEF)
2156 DefinedValues.insert(Node->getOperand(i));
2163 SDValue V = Node->getOperand(i);
2168 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
2182 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2188 return ExpandVectorBuildThroughStack(Node);
2191 // Expand a node into a call to a libcall. If the result value
2195 SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2199 for (const SDValue &Op : Node->op_values()) {
2202 Entry.Node = Op;
2211 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2213 // By default, the input chain to this libcall is the entry node of the
2216 // node which is being folded has a non-entry input chain.
2222 bool isTailCall = TLI.isInTailCallPosition(DAG, Node, TCChain);
2227 CLI.setDebugLoc(SDLoc(Node)).setChain(InChain)
2250 Entry.Node = Ops[i];
2251 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
2271 // Expand a node into a call to a libcall. Similar to
2275 SDNode *Node,
2277 SDValue InChain = Node->getOperand(0);
2281 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
2282 EVT ArgVT = Node->getOperand(i).getValueType();
2284 Entry.Node = Node->getOperand(i);
2293 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2296 CLI.setDebugLoc(SDLoc(Node)).setChain(InChain)
2305 SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2312 switch (Node->getSimpleValueType(0).SimpleTy) {
2320 return ExpandLibCall(LC, Node, false);
2323 SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2330 switch (Node->getSimpleValueType(0).SimpleTy) {
2338 return ExpandLibCall(LC, Node, isSigned);
2343 SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2345 unsigned Opcode = Node->getOpcode();
2349 switch (Node->getSimpleValueType(0).SimpleTy) {
2358 // The input chain to this libcall is the entry node of the function.
2363 EVT RetVT = Node->getValueType(0);
2368 for (const SDValue &Op : Node->op_values()) {
2371 Entry.Node = Op;
2380 Entry.Node = FIPtr;
2389 SDLoc dl(Node);
2405 static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2407 switch (Node->getSimpleValueType(0).SimpleTy) {
2420 static bool canCombineSinCosLibcall(SDNode *Node, const TargetLowering &TLI,
2422 if (!isSinCosLibcallAvailable(Node, TLI))
2433 static bool useSinCos(SDNode *Node) {
2434 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2437 SDValue Op0 = Node->getOperand(0);
2441 if (User == Node)
2452 SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2455 switch (Node->getSimpleValueType(0).SimpleTy) {
2464 // The input chain to this libcall is the entry node of the function.
2469 EVT RetVT = Node->getValueType(0);
2476 Entry.Node = Node->getOperand(0);
2484 Entry.Node = SinPtr;
2492 Entry.Node = CosPtr;
2501 SDLoc dl(Node);
2988 bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2990 SDLoc dl(Node);
2993 switch (Node->getOpcode()) {
2999 Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl);
3003 Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
3006 Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
3011 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3014 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
3023 Results.push_back(Node->getOperand(0));
3028 Results.append(Node->getNumValues() - 1,
3029 DAG.getConstant(0, dl, Node->getValueType(0)));
3030 Results.push_back(Node->getOperand(0));
3036 Results.push_back(Node->getOperand(0));
3040 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
3041 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3043 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
3044 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
3045 cast<AtomicSDNode>(Node)->getMemOperand(),
3046 cast<AtomicSDNode>(Node)->getOrdering(),
3047 cast<AtomicSDNode>(Node)->getOrdering(),
3048 cast<AtomicSDNode>(Node)->getSynchScope());
3056 cast<AtomicSDNode>(Node)->getMemoryVT(),
3057 Node->getOperand(0),
3058 Node->getOperand(1), Node->getOperand(2),
3059 cast<AtomicSDNode>(Node)->getMemOperand(),
3060 cast<AtomicSDNode>(Node)->getOrdering(),
3061 cast<AtomicSDNode>(Node)->getSynchScope());
3069 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3071 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
3072 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
3073 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand(),
3074 cast<AtomicSDNode>(Node)->getSuccessOrdering(),
3075 cast<AtomicSDNode>(Node)->getFailureOrdering(),
3076 cast<AtomicSDNode>(Node)->getSynchScope());
3078 SDValue Success = DAG.getSetCC(SDLoc(Node), Node->getValueType(1),
3079 Res, Node->getOperand(2), ISD::SETEQ);
3087 ExpandDYNAMIC_STACKALLOC(Node, Results);
3090 for (unsigned i = 0; i < Node->getNumValues(); i++)
3091 Results.push_back(Node->getOperand(i));
3094 EVT VT = Node->getValueType(0);
3105 Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
3106 Node->getValueType(0), dl);
3110 Tmp1 = EmitStackConvert(Node->getOperand(0),
3111 Node->getOperand(0).getValueType(),
3112 Node->getValueType(0), dl);
3118 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3119 EVT VT = Node->getValueType(0);
3126 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
3127 Node->getOperand(0), ShiftCst);
3128 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
3139 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3140 Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
3141 Node->getValueType(0), dl);
3147 Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
3148 Node->getOperand(0), Node->getValueType(0), dl);
3152 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
3157 EVT VT = Node->getOperand(0).getValueType();
3158 EVT NVT = Node->getValueType(0);
3165 Node->getOperand(0),
3167 True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0));
3171 Node->getOperand(0), Tmp1));
3179 Results.push_back(DAG.expandVAArg(Node));
3183 Results.push_back(DAG.expandVACopy(Node));
3186 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
3188 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
3189 Node->getOperand(0));
3191 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3195 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3198 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3201 Results.push_back(ExpandVectorBuildThroughStack(Node));
3205 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3208 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3209 Node->getOperand(1),
3210 Node->getOperand(2), dl));
3214 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3216 EVT VT = Node->getValueType(0);
3218 SDValue Op0 = Node->getOperand(0);
3219 SDValue Op1 = Node->getOperand(1);
3229 // Convert shuffle node.
3230 // If original node was v4i64 and the new EltVT is i32,
3285 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3286 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3291 EVT OpTy = Node->getOperand(0).getValueType();
3292 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
3294 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3297 Node->getOperand(0).getValueType(),
3299 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3302 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3303 Node->getOperand(0));
3312 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3313 Node->getValueType(0)));
3316 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3317 Results.push_back(Node->getOperand(0));
3324 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3325 Node->getOperand(1)));
3327 Results.push_back(Node->getOperand(0));
3331 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3335 Results.push_back(ExpandFCOPYSIGN(Node));
3339 Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
3341 Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
3342 Node->getOperand(0));
3346 Results.push_back(ExpandFABS(Node));
3354 switch (Node->getOpcode()) {
3361 Tmp1 = Node->getOperand(0);
3362 Tmp2 = Node->getOperand(1);
3370 EVT VT = Node->getValueType(0);
3371 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3374 canCombineSinCosLibcall(Node, TLI, TM))
3375 && useSinCos(Node)) {
3377 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3378 if (Node->getOpcode() == ISD::FCOS)
3388 if (Node->getValueType(0) != MVT::f32) {
3393 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3395 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3400 SDValue Op = Node->getOperand(0);
3404 // Under fastmath, we can expand this node into a fround followed by
3414 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3416 // If this is a legal constant, turn it into a TargetConstantFP node.
3417 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0)))
3422 ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3427 EVT VT = Node->getValueType(0);
3430 const SDNodeFlags *Flags = &cast<BinaryWithFlagsSDNode>(Node)->Flags;
3431 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3432 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3438 EVT VT = Node->getValueType(0);
3442 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3446 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3451 EVT VT = Node->getValueType(0);
3452 bool isSigned = Node->getOpcode() == ISD::SREM;
3455 Tmp2 = Node->getOperand(0);
3456 Tmp3 = Node->getOperand(1);
3472 bool isSigned = Node->getOpcode() == ISD::SDIV;
3474 EVT VT = Node->getValueType(0);
3477 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3478 Node->getOperand(1));
3485 unsigned ExpandOpcode = Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI :
3487 EVT VT = Node->getValueType(0);
3491 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3492 Node->getOperand(1));
3497 EVT VT = Node->getValueType(0);
3519 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3520 Node->getOperand(1)));
3530 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG)) {
3543 SDValue LHS = Node->getOperand(0);
3544 SDValue RHS = Node->getOperand(1);
3545 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3549 EVT ResultType = Node->getValueType(1);
3550 EVT OType = getSetCCResultType(Node->getValueType(0));
3566 Node->getOpcode() == ISD::SADDO ?
3578 SDValue LHS = Node->getOperand(0);
3579 SDValue RHS = Node->getOperand(1);
3580 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ?
3585 EVT ResultType = Node->getValueType(1);
3586 EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3588 = Node->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
3596 EVT VT = Node->getValueType(0);
3598 SDValue LHS = Node->getOperand(0);
3599 SDValue RHS = Node->getOperand(1);
3605 bool isSigned = Node->getOpcode() == ISD::SMULO;
3659 // Ret is a node with an illegal type. Because such things are not
3661 // node has no more uses. The above EXTRACT_ELEMENT nodes should have been
3683 EVT PairTy = Node->getValueType(0);
3684 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3685 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3694 Tmp1 = Node->getOperand(0);
3695 Tmp2 = Node->getOperand(1);
3696 Tmp3 = Node->getOperand(2);
3709 SDValue Chain = Node->getOperand(0);
3710 SDValue Table = Node->getOperand(1);
3711 SDValue Index = Node->getOperand(2);
3743 // Node.
3744 Tmp1 = Node->getOperand(0);
3745 Tmp2 = Node->getOperand(1);
3750 Node->getOperand(2));
3759 Node->getOperand(2));
3764 Tmp1 = Node->getOperand(0);
3765 Tmp2 = Node->getOperand(1);
3766 Tmp3 = Node->getOperand(2);
3767 bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2,
3772 // condition code, create a new SETCC node.
3774 Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3788 EVT VT = Node->getValueType(0);
3807 Tmp1 = Node->getOperand(0); // LHS
3808 Tmp2 = Node->getOperand(1); // RHS
3809 Tmp3 = Node->getOperand(2); // True
3810 Tmp4 = Node->getOperand(3); // False
3811 EVT VT = Node->getValueType(0);
3812 SDValue CC = Node->getOperand(4);
3817 // node using SETCC and SELECT.
3865 // condition code, create a new SELECT_CC node.
3867 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3872 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3880 Tmp1 = Node->getOperand(0); // Chain
3881 Tmp2 = Node->getOperand(2); // LHS
3882 Tmp3 = Node->getOperand(3); // RHS
3883 Tmp4 = Node->getOperand(1); // CC
3896 // node.
3898 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3899 Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3903 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3904 Tmp2, Tmp3, Node->getOperand(4));
3910 Results.push_back(ExpandBUILD_VECTOR(Node));
3916 EVT VT = Node->getValueType(0);
3924 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
3927 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
3929 Node->getOpcode(), dl,
3933 DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0), Scalars);
3934 ReplaceNode(SDValue(Node, 0), Result);
3950 // Replace the original node with the legalized result.
3954 ReplaceNode(Node, Results.data());
3958 void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3960 SDLoc dl(Node);
3962 unsigned Opc = Node->getOpcode();
3971 .setChain(Node->getOperand(0))
3997 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
4001 std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
4011 .setChain(Node->getOperand(0))
4022 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4027 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4032 Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4037 Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4042 Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
4048 ExpandSinCosLibCall(Node, Results);
4051 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
4056 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
4061 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
4066 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
4071 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
4076 Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4081 Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4086 Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4091 Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4096 Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4103 Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4110 Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
4115 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
4120 Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4125 Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4130 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4135 Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4140 Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4145 if (Node->getValueType(0) == MVT::f32) {
4146 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
4151 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4153 Results.push_back(ExpandLibCall(LC, Node, false));
4157 Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4162 Results.push_back(ExpandIntLibCall(Node, true,
4168 Results.push_back(ExpandIntLibCall(Node, false,
4174 Results.push_back(ExpandIntLibCall(Node, true,
4180 Results.push_back(ExpandIntLibCall(Node, false,
4188 ExpandDivRemLibCall(Node, Results);
4191 Results.push_back(ExpandIntLibCall(Node, false,
4198 // Replace the original node with the legalized result.
4200 ReplaceNode(Node, Results.data());
4213 void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4215 MVT OVT = Node->getSimpleValueType(0);
4216 if (Node->getOpcode() == ISD::UINT_TO_FP ||
4217 Node->getOpcode() == ISD::SINT_TO_FP ||
4218 Node->getOpcode() == ISD::SETCC ||
4219 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4220 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4221 OVT = Node->getOperand(0).getSimpleValueType();
4223 if (Node->getOpcode() == ISD::BR_CC)
4224 OVT = Node->getOperand(2).getSimpleValueType();
4225 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4226 SDLoc dl(Node);
4228 switch (Node->getOpcode()) {
4235 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4238 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4239 if (Node->getOpcode() == ISD::CTTZ) {
4246 } else if (Node->getOpcode() == ISD::CTLZ ||
4247 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4257 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4268 Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
4269 Node->getOpcode() == ISD::FP_TO_SINT, dl);
4274 Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
4275 Node->getOpcode() == ISD::SINT_TO_FP, dl);
4279 SDValue Chain = Node->getOperand(0); // Get the chain.
4280 SDValue Ptr = Node->getOperand(1); // Get the pointer.
4292 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4293 Node->getConstantOperandVal(3));
4300 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4301 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4306 ReplacedNode(Node);
4322 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4323 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4325 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4331 if (Node->getValueType(0).isVector() ||
4332 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4335 } else if (Node->getValueType(0).isInteger()) {
4342 Tmp1 = Node->getOperand(0);
4344 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4345 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4349 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4351 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4357 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4360 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4361 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4373 cast<CondCodeSDNode>(Node->getOperand(2))->get();
4376 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4377 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4378 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
4379 Tmp1, Tmp2, Node->getOperand(2)));
4386 cast<CondCodeSDNode>(Node->getOperand(1))->get();
4389 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4390 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4391 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4392 Node->getOperand(0), Node->getOperand(1),
4393 Tmp1, Tmp2, Node->getOperand(4)));
4404 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4405 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4406 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4407 Node->getFlags());
4413 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4414 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4415 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4418 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4424 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4425 Tmp2 = Node->getOperand(1);
4426 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4433 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4454 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4455 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4477 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
4478 SDValue Op = Node->getOperand(I);
4482 SDLoc SL(Node);
4510 SDValue Idx = Node->getOperand(1);
4512 SDLoc SL(Node);
4516 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4557 SDValue Val = Node->getOperand(1);
4558 SDValue Idx = Node->getOperand(2);
4560 SDLoc SL(Node);
4565 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4595 SDValue Val = Node->getOperand(0);
4596 SDLoc SL(Node);
4613 // Replace the original node with the legalized result.
4615 ReplaceNode(Node, Results.data());
4665 // Directly insert the node in question, and legalize it. This will recurse
